MOW-118: switch from KV to DO for caching

pull/139/head
Sven Sauleau 2023-01-18 13:31:50 +00:00
rodzic 346b0a19dc
commit 9c11b74c2e
20 zmienionych plików z 340 dodań i 98 usunięć

Wyświetl plik

@ -166,6 +166,20 @@ jobs:
echo "VAPID keys generated"
fi
- name: Publish DO
uses: cloudflare/wrangler-action@2.0.0
with:
apiToken: ${{ secrets.CF_API_TOKEN }}
command: publish --config do/wrangler.toml
env:
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
- name: Retrieve DO namespace
run: |
curl https://api.cloudflare.com/client/v4/accounts/${{ secrets.CF_ACCOUNT_ID }}/workers/durable_objects/namespaces \
-H 'Authorization: Bearer ${{ secrets.CF_API_TOKEN }}' \
| jq -r '.result[] | select( .script == "wildebeest-do" ) | .id' | awk '{print "do_cache_id="$1}' >> $GITHUB_ENV
- name: Configure
run: terraform plan && terraform apply -auto-approve
continue-on-error: true
@ -177,6 +191,7 @@ jobs:
TF_VAR_cloudflare_deploy_domain: ${{ vars.CF_DEPLOY_DOMAIN }}
TF_VAR_gh_username: ${{ env.OWNER_LOWER }}
TF_VAR_d1_id: ${{ env.d1_id }}
TF_VAR_do_cache_id: ${{ env.do_cache_id }}
TF_VAR_access_auth_domain: ${{ env.auth_domain }}
TF_VAR_wd_instance_title: ${{ vars.INSTANCE_TITLE }}
TF_VAR_wd_admin_email: ${{ vars.ADMIN_EMAIL }}
@ -214,18 +229,6 @@ jobs:
env:
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
- name: retrieve Wildebeest cache KV namespace
uses: cloudflare/wrangler-action@2.0.0
with:
command: kv:namespace list | jq -r '.[] | select( .title == "wildebeest-${{ env.OWNER_LOWER }}-cache" ) | .id' | awk '{print "cache_kv="$1}' >> $GITHUB_ENV
apiToken: ${{ secrets.CF_API_TOKEN }}
preCommands: |
echo "*** pre commands ***"
apt-get update && apt-get -y install jq
echo "******"
env:
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
- name: Create Queue
uses: cloudflare/wrangler-action@2.0.0
with:
@ -243,9 +246,10 @@ jobs:
echo "*** pre commands ***"
echo -e "[[d1_databases]]\nbinding=\"DATABASE\"\ndatabase_name=\"wildebeest-${{ env.OWNER_LOWER }}\"\ndatabase_id=\"${{ env.d1_id }}\"\n" >> consumer/wrangler.toml
echo -e "[[kv_namespaces]]\n" >> consumer/wrangler.toml
echo -e "binding=\"KV_CACHE\"\n" >> consumer/wrangler.toml
echo -e "id=\"${{ env.cache_kv }}\"\n" >> consumer/wrangler.toml
echo -e "[durable_objects]\n" >> consumer/wrangler.toml
echo -e "bindings=[" >> consumer/wrangler.toml
echo -e "{name=\"DO_CACHE\",class_name=\"WildebeestCache\",script_name=\"wildebeest-do\"}," >> consumer/wrangler.toml
echo -e "]" >> consumer/wrangler.toml
echo -e "[vars]\n" >> consumer/wrangler.toml
echo -e "DOMAIN=\"${{ vars.CF_DEPLOY_DOMAIN }}\"\n" >> consumer/wrangler.toml

36
backend/src/cache/index.ts vendored 100644
Wyświetl plik

@ -0,0 +1,36 @@
const CACHE_DO_NAME = 'cachev1'
export interface Cache {
get<T>(key: string): Promise<T | null>
put<T>(key: string, value: T): Promise<void>
}
export function cacheFromEnv(env: any): Cache {
return {
async get<T>(key: string): Promise<T | null> {
const id = env.DO_CACHE.idFromName(CACHE_DO_NAME)
const stub = env.DO_CACHE.get(id)
const res = await stub.fetch('http://cache/' + key)
if (!res.ok) {
throw new Error(`DO cache returned ${res.status}: ${await res.text()}`)
}
return (await res.json()) as T
},
async put<T>(key: string, value: T): Promise<void> {
const id = env.DO_CACHE.idFromName(CACHE_DO_NAME)
const stub = env.DO_CACHE.get(id)
const res = await stub.fetch('http://cache/', {
method: 'PUT',
body: JSON.stringify({ key, value }),
})
if (!res.ok) {
throw new Error(`DO cache returned ${res.status}: ${await res.text()}`)
}
},
}
}

Wyświetl plik

@ -10,6 +10,7 @@ import { WebPushResult } from 'wildebeest/backend/src/webpush/webpushinfos'
import type { Actor } from 'wildebeest/backend/src/activitypub/actors'
import type { NotificationType, Notification } from 'wildebeest/backend/src/types/notification'
import { getSubscriptionForAllClients } from 'wildebeest/backend/src/mastodon/subscription'
import type { Cache } from 'wildebeest/backend/src/cache'
export async function createNotification(
db: D1Database,
@ -250,7 +251,7 @@ export async function getNotifications(db: D1Database, actor: Actor, domain: str
return out
}
export async function pregenerateNotifications(db: D1Database, cache: KVNamespace, actor: Actor, domain: string) {
export async function pregenerateNotifications(db: D1Database, cache: Cache, actor: Actor, domain: string) {
const notifications = await getNotifications(db, actor, domain)
await cache.put(actor.id + '/notifications', JSON.stringify(notifications))
await cache.put(actor.id + '/notifications', notifications)
}

Wyświetl plik

@ -3,10 +3,11 @@ import { getFollowingId } from 'wildebeest/backend/src/mastodon/follow'
import type { Actor } from 'wildebeest/backend/src/activitypub/actors/'
import { toMastodonStatusFromRow } from './status'
import { PUBLIC_GROUP } from 'wildebeest/backend/src/activitypub/activities'
import type { Cache } from 'wildebeest/backend/src/cache'
export async function pregenerateTimelines(domain: string, db: D1Database, cache: KVNamespace, actor: Actor) {
export async function pregenerateTimelines(domain: string, db: D1Database, cache: Cache, actor: Actor) {
const timeline = await getHomeTimeline(domain, db, actor)
await cache.put(actor.id + '/timeline/home', JSON.stringify(timeline))
await cache.put(actor.id + '/timeline/home', timeline)
}
export async function getHomeTimeline(domain: string, db: D1Database, actor: Actor): Promise<Array<MastodonStatus>> {

Wyświetl plik

@ -2,9 +2,10 @@ import type { Queue, MessageBody } from 'wildebeest/backend/src/types/queue'
export interface Env {
DATABASE: D1Database
KV_CACHE: KVNamespace
// FIXME: shouldn't it be USER_KEY?
userKEK: string
QUEUE: Queue<MessageBody>
DO_CACHE: DurableObjectNamespace
CF_ACCOUNT_ID: string
CF_API_TOKEN: string

Wyświetl plik

@ -4,7 +4,7 @@ import { createPublicNote } from 'wildebeest/backend/src/activitypub/objects/not
import { createNotification, insertFollowNotification } from 'wildebeest/backend/src/mastodon/notification'
import { createPerson } from 'wildebeest/backend/src/activitypub/actors'
import * as notifications from 'wildebeest/functions/api/v1/notifications'
import { makeDB, assertJSON, createTestClient } from '../utils'
import { makeCache, makeDB, assertJSON, createTestClient } from '../utils'
import { strict as assert } from 'node:assert/strict'
import { sendLikeNotification } from 'wildebeest/backend/src/mastodon/notification'
import { createSubscription } from 'wildebeest/backend/src/mastodon/subscription'
@ -32,15 +32,13 @@ describe('Mastodon APIs', () => {
test('returns notifications stored in KV cache', async () => {
const db = await makeDB()
const connectedActor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com')
const kv_cache: any = {
async get(key: string) {
assert.equal(key, connectedActor.id + '/notifications')
return 'cached data'
},
}
const cache = makeCache()
await cache.put(connectedActor.id + '/notifications', 12345)
const req = new Request('https://' + domain)
const data = await notifications.handleRequest(req, kv_cache, connectedActor)
assert.equal(await data.text(), 'cached data')
const data = await notifications.handleRequest(req, cache, connectedActor)
assert.equal(await data.json(), 12345)
})
test('returns notifications stored in db', async () => {

Wyświetl plik

@ -12,7 +12,7 @@ import * as statuses_context from 'wildebeest/functions/api/v1/statuses/[id]/con
import { createPerson } from 'wildebeest/backend/src/activitypub/actors'
import { insertLike } from 'wildebeest/backend/src/mastodon/like'
import { insertReblog } from 'wildebeest/backend/src/mastodon/reblog'
import { isUrlValid, makeDB, assertJSON, streamToArrayBuffer, makeQueue } from '../utils'
import { isUrlValid, makeDB, assertJSON, streamToArrayBuffer, makeQueue, makeCache } from '../utils'
import * as activities from 'wildebeest/backend/src/activitypub/activities'
import { addFollowing, acceptFollowing } from 'wildebeest/backend/src/mastodon/follow'
import { MessageType } from 'wildebeest/backend/src/types/queue'
@ -20,9 +20,7 @@ import { MessageType } from 'wildebeest/backend/src/types/queue'
const userKEK = 'test_kek4'
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
const domain = 'cloudflare.com'
const kv_cache: any = {
async put() {},
}
const cache = makeCache()
describe('Mastodon APIs', () => {
describe('statuses', () => {
@ -38,7 +36,7 @@ describe('Mastodon APIs', () => {
})
const connectedActor: any = {}
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache)
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache)
assert.equal(res.status, 400)
})
@ -58,7 +56,7 @@ describe('Mastodon APIs', () => {
})
const connectedActor = actor
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache)
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache)
assert.equal(res.status, 200)
assertJSON(res)
@ -96,14 +94,7 @@ describe('Mastodon APIs', () => {
const db = await makeDB()
const queue = makeQueue()
const actor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com')
let cache = null
const kv_cache: any = {
async put(key: string, value: any) {
assert.equal(key, actor.id + '/timeline/home')
cache = value
},
}
const cache = makeCache()
const body = {
status: 'my status',
@ -115,16 +106,17 @@ describe('Mastodon APIs', () => {
body: JSON.stringify(body),
})
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache)
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache)
assert.equal(res.status, 200)
assertJSON(res)
const data = await res.json<any>()
assert(cache)
const timeline = JSON.parse(cache)
assert.equal(timeline.length, 1)
assert.equal(timeline[0].id, data.id)
const cachedData = await cache.get<any>(actor.id + '/timeline/home')
console.log({ cachedData })
assert(cachedData)
assert.equal(cachedData.length, 1)
assert.equal(cachedData[0].id, data.id)
})
test("create new status adds to Actor's outbox", async () => {
@ -143,7 +135,7 @@ describe('Mastodon APIs', () => {
})
const connectedActor = actor
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache)
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache)
assert.equal(res.status, 200)
const row = await db.prepare(`SELECT count(*) as count FROM outbox_objects`).first()
@ -171,7 +163,7 @@ describe('Mastodon APIs', () => {
body: JSON.stringify(body),
})
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache)
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache)
assert.equal(res.status, 200)
assert.equal(queue.messages.length, 2)
@ -250,7 +242,7 @@ describe('Mastodon APIs', () => {
})
const connectedActor = actor
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache)
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache)
assert.equal(res.status, 200)
assert(deliveredNote)
@ -281,7 +273,7 @@ describe('Mastodon APIs', () => {
body: JSON.stringify(body),
})
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache)
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache)
assert.equal(res.status, 200)
const data = await res.json<any>()
@ -560,7 +552,7 @@ describe('Mastodon APIs', () => {
body: JSON.stringify(body),
})
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache)
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache)
assert.equal(res.status, 404)
})
@ -581,7 +573,7 @@ describe('Mastodon APIs', () => {
body: JSON.stringify(body),
})
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache)
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache)
assert.equal(res.status, 200)
const data = await res.json<any>()
@ -625,7 +617,7 @@ describe('Mastodon APIs', () => {
body: JSON.stringify(body),
})
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache)
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache)
assert.equal(res.status, 400)
const data = await res.json<any>()
assert(data.error.includes('Limit exceeded'))
@ -650,7 +642,7 @@ describe('Mastodon APIs', () => {
body,
})
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache)
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache)
assert.equal(res.status, 400)
const data = await res.json<any>()
assert(data.error.includes('Limit exceeded'))

Wyświetl plik

@ -5,7 +5,7 @@ import { addFollowing, acceptFollowing } from 'wildebeest/backend/src/mastodon/f
import { createPublicNote, createPrivateNote } from 'wildebeest/backend/src/activitypub/objects/note'
import { addObjectInOutbox } from 'wildebeest/backend/src/activitypub/actors/outbox'
import { createPerson } from 'wildebeest/backend/src/activitypub/actors'
import { makeDB, assertCORS, assertJSON } from '../utils'
import { makeDB, assertCORS, assertJSON, makeCache } from '../utils'
import * as timelines_home from 'wildebeest/functions/api/v1/timelines/home'
import * as timelines_public from 'wildebeest/functions/api/v1/timelines/public'
import * as timelines from 'wildebeest/backend/src/mastodon/timeline'
@ -106,27 +106,20 @@ describe('Mastodon APIs', () => {
test('home returns cache', async () => {
const db = await makeDB()
const connectedActor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com')
const kv_cache: any = {
async get(key: string) {
assert.equal(key, connectedActor.id + '/timeline/home')
return 'cached data'
},
}
const cache = makeCache()
await cache.put(connectedActor.id + '/timeline/home', 12345)
const req = new Request('https://' + domain)
const data = await timelines_home.handleRequest(req, kv_cache, connectedActor)
assert.equal(await data.text(), 'cached data')
const data = await timelines_home.handleRequest(req, cache, connectedActor)
assert.equal(await data.json(), 12345)
})
test('home returns empty if not in cache', async () => {
const db = await makeDB()
const connectedActor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com')
const kv_cache: any = {
async get() {
return null
},
}
const cache = makeCache()
const req = new Request('https://' + domain)
const data = await timelines_home.handleRequest(req, kv_cache, connectedActor)
const data = await timelines_home.handleRequest(req, cache, connectedActor)
const posts = await data.json<Array<any>>()
assert.equal(posts.length, 0)

Wyświetl plik

@ -1,4 +1,5 @@
import { strict as assert } from 'node:assert/strict'
import type { Cache } from 'wildebeest/backend/src/cache'
import type { Queue } from 'wildebeest/backend/src/types/queue'
import { createClient } from 'wildebeest/backend/src/mastodon/client'
import type { Client } from 'wildebeest/backend/src/mastodon/client'
@ -91,6 +92,24 @@ export function makeQueue(): TestQueue {
}
}
export function makeCache(): Cache {
const cache: any = {}
return {
async get<T>(key: string): Promise<T | null> {
if (cache[key]) {
return cache[key] as T
} else {
return null
}
},
async put<T>(key: string, value: T): Promise<void> {
cache[key] = value
},
}
}
export function isUUID(v: string): boolean {
assert.equal(typeof v, 'string')
if (v.match('^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$') === null) {

Wyświetl plik

@ -2,6 +2,7 @@ import type { MessageBody, InboxMessageBody } from 'wildebeest/backend/src/types
import * as activityHandler from 'wildebeest/backend/src/activitypub/activities/handle'
import * as notification from 'wildebeest/backend/src/mastodon/notification'
import * as timeline from 'wildebeest/backend/src/mastodon/timeline'
import { cacheFromEnv } from 'wildebeest/backend/src/cache'
import type { Actor } from 'wildebeest/backend/src/activitypub/actors'
import type { Env } from './'
@ -9,7 +10,7 @@ export async function handleInboxMessage(env: Env, actor: Actor, message: InboxM
const domain = env.DOMAIN
const db = env.DATABASE
const adminEmail = env.ADMIN_EMAIL
const cache = env.KV_CACHE
const cache = cacheFromEnv(env)
const activity = message.activity
await activityHandler.handle(domain, activity, db, message.userKEK, adminEmail, message.vapidKeys)

Wyświetl plik

@ -12,7 +12,7 @@ export type Env = {
DATABASE: D1Database
DOMAIN: string
ADMIN_EMAIL: string
KV_CACHE: KVNamespace
DO_CACHE: DurableObjectNamespace
}
export default {

14
do/package.json 100644
Wyświetl plik

@ -0,0 +1,14 @@
{
"name": "wildebeest-do",
"version": "0.0.0",
"devDependencies": {
"@cloudflare/workers-types": "^4.20221111.1",
"typescript": "^4.9.4",
"wrangler": "2.7.1"
},
"private": true,
"scripts": {
"start": "wrangler dev",
"deploy": "wrangler publish"
}
}

49
do/src/index.ts 100644
Wyświetl plik

@ -0,0 +1,49 @@
export interface Env {
DO: DurableObjectNamespace
}
export default {
async fetch(request: Request, env: Env) {
try {
const id = env.DO.idFromName('default')
const obj = env.DO.get(id)
return obj.fetch(request)
} catch (err: any) {
return new Response(err.stack, { status: 500 })
}
},
}
export class WildebeestCache {
private storage: DurableObjectStorage
constructor(state: DurableObjectState) {
this.storage = state.storage
}
async fetch(request: Request) {
if (request.method === 'GET') {
const { pathname } = new URL(request.url)
const key = pathname.slice(1) // remove the leading slash from path
const value = await this.storage.get(key)
if (value === undefined) {
console.log(`Get ${key} MISS`)
return new Response('', { status: 404 })
}
console.log(`Get ${key} HIT`)
return new Response(JSON.stringify(value))
}
if (request.method === 'PUT') {
const { key, value } = await request.json<any>()
console.log(`Set ${key}`)
await this.storage.put(key, value)
return new Response('', { status: 201 })
}
return new Response('', { status: 400 })
}
}

106
do/tsconfig.json 100644
Wyświetl plik

@ -0,0 +1,106 @@
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig.json to read more about this file */
/* Projects */
// "incremental": true, /* Enable incremental compilation */
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
// "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
/* Language and Environment */
"target": "es2021" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
"lib": [
"es2021"
] /* Specify a set of bundled library declaration files that describe the target runtime environment. */,
"jsx": "react" /* Specify what JSX code is generated. */,
// "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */
// "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */
// "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
/* Modules */
"module": "es2022" /* Specify what module code is generated. */,
// "rootDir": "./", /* Specify the root folder within your source files. */
"moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */,
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
// "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */
"types": [
"@cloudflare/workers-types",
"jest"
] /* Specify type package names to be included without being referenced in a source file. */,
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
"resolveJsonModule": true /* Enable importing .json files */,
// "noResolve": true, /* Disallow `import`s, `require`s or `<reference>`s from expanding the number of files TypeScript should add to a project. */
/* JavaScript Support */
"allowJs": true /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */,
"checkJs": false /* Enable error reporting in type-checked JavaScript files. */,
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */
/* Emit */
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
// "sourceMap": true, /* Create source map files for emitted JavaScript files. */
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */
// "outDir": "./", /* Specify an output folder for all emitted files. */
// "removeComments": true, /* Disable emitting comments. */
"noEmit": true /* Disable emitting files from a compilation. */,
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
// "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
// "newLine": "crlf", /* Set the newline character for emitting files. */
// "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */
// "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
// "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
// "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
/* Interop Constraints */
"isolatedModules": true /* Ensure that each file can be safely transpiled without relying on other imports. */,
"allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */,
// "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */,
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
/* Type Checking */
"strict": true /* Enable all strict type-checking options. */,
// "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */
// "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */
// "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
// "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */
// "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */
// "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */
// "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */
// "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */
// "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */
// "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */
// "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
// "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
// "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
// "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
// "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
/* Completeness */
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
"skipLibCheck": true /* Skip type checking all .d.ts files. */
}
}

7
do/wrangler.toml 100644
Wyświetl plik

@ -0,0 +1,7 @@
name = "wildebeest-do"
main = "src/index.ts"
compatibility_date = "2023-01-18"
[[migrations]]
tag = "v1"
new_classes = ["WildebeestCache"]

Wyświetl plik

@ -6,14 +6,15 @@ import type { Account, MastodonStatus } from 'wildebeest/frontend/src/types'
const kek = 'test-kek'
/* eslint-disable @typescript-eslint/no-empty-function */
const queue = {
async send() {},
async sendBatch() {},
const queue: unknown = {
send() {},
sendBatch() {},
}
const kv_cache = {
async put() {},
/* eslint-disable @typescript-eslint/no-empty-function */
const cache: unknown = {
get() {},
put() {},
}
/* eslint-enable @typescript-eslint/no-empty-function */
/**
* Run helper commands to initialize the database with actors, statuses, etc.
@ -30,7 +31,7 @@ export async function init(domain: string, db: D1Database) {
const reblogger = await getOrCreatePerson(domain, db, rebloggerAccount)
// Reblog an arbitrary status with this reblogger
const statusToReblog = loadedStatuses[2]
await reblogStatus(db, reblogger, statusToReblog)
await reblogStatus(db, reblogger, statusToReblog, domain)
}
/**
@ -49,7 +50,7 @@ async function createStatus(db: D1Database, actor: Person, status: string, visib
headers,
body: JSON.stringify(body),
})
const resp = await statusesAPI.handleRequest(req, db, actor, kek, queue, kv_cache as unknown as KVNamespace)
const resp = await statusesAPI.handleRequest(req, db, actor, kek, queue, cache)
return (await resp.json()) as MastodonStatus
}
@ -70,6 +71,6 @@ async function getOrCreatePerson(
return newPerson
}
async function reblogStatus(db: D1Database, actor: Person, status: MastodonStatus) {
await reblogAPI.handleRequest(db, status.id, actor, kek, queue)
async function reblogStatus(db: D1Database, actor: Person, status: MastodonStatus, domain: string) {
await reblogAPI.handleRequest(db, status.id, actor, kek, queue, domain)
}

Wyświetl plik

@ -4,9 +4,11 @@ import { cors } from 'wildebeest/backend/src/utils/cors'
import type { Env } from 'wildebeest/backend/src/types/env'
import type { Person } from 'wildebeest/backend/src/activitypub/actors'
import type { ContextData } from 'wildebeest/backend/src/types/context'
import type { Cache } from 'wildebeest/backend/src/cache'
import { cacheFromEnv } from 'wildebeest/backend/src/cache'
export const onRequest: PagesFunction<Env, any, ContextData> = async ({ request, env, data }) => {
return handleRequest(request, env.KV_CACHE, data.connectedActor)
return handleRequest(request, cacheFromEnv(env), data.connectedActor)
}
const headers = {
@ -14,7 +16,7 @@ const headers = {
'content-type': 'application/json; charset=utf-8',
}
export async function handleRequest(request: Request, cache: KVNamespace, connectedActor: Person): Promise<Response> {
export async function handleRequest(request: Request, cache: Cache, connectedActor: Person): Promise<Response> {
const url = new URL(request.url)
if (url.searchParams.has('max_id')) {
// We just return the pregenerated notifications, without any filter for
@ -23,9 +25,9 @@ export async function handleRequest(request: Request, cache: KVNamespace, connec
return new Response(JSON.stringify([]), { headers })
}
const notifications = await cache.get(connectedActor.id + '/notifications')
const notifications = await cache.get<any>(connectedActor.id + '/notifications')
if (notifications === null) {
return new Response(JSON.stringify([]), { headers })
}
return new Response(notifications, { headers })
return new Response(JSON.stringify(notifications), { headers })
}

Wyświetl plik

@ -21,6 +21,8 @@ import { readBody } from 'wildebeest/backend/src/utils/body'
import * as errors from 'wildebeest/backend/src/errors'
import type { Visibility } from 'wildebeest/backend/src/types'
import { toMastodonStatusFromObject } from 'wildebeest/backend/src/mastodon/status'
import type { Cache } from 'wildebeest/backend/src/cache'
import { cacheFromEnv } from 'wildebeest/backend/src/cache'
type StatusCreate = {
status: string
@ -31,7 +33,7 @@ type StatusCreate = {
}
export const onRequest: PagesFunction<Env, any, ContextData> = async ({ request, env, data }) => {
return handleRequest(request, env.DATABASE, data.connectedActor, env.userKEK, env.QUEUE, env.KV_CACHE)
return handleRequest(request, env.DATABASE, data.connectedActor, env.userKEK, env.QUEUE, cacheFromEnv(env))
}
// FIXME: add tests for delivery to followers and mentions to a specific Actor.
@ -41,7 +43,7 @@ export async function handleRequest(
connectedActor: Person,
userKEK: string,
queue: Queue<DeliverMessageBody>,
cache: KVNamespace
cache: Cache
): Promise<Response> {
// TODO: implement Idempotency-Key

Wyświetl plik

@ -2,6 +2,8 @@ import type { Env } from 'wildebeest/backend/src/types/env'
import { cors } from 'wildebeest/backend/src/utils/cors'
import type { ContextData } from 'wildebeest/backend/src/types/context'
import type { Actor } from 'wildebeest/backend/src/activitypub/actors/'
import type { Cache } from 'wildebeest/backend/src/cache'
import { cacheFromEnv } from 'wildebeest/backend/src/cache'
const headers = {
...cors(),
@ -9,10 +11,10 @@ const headers = {
}
export const onRequest: PagesFunction<Env, any, ContextData> = async ({ request, env, data }) => {
return handleRequest(request, env.KV_CACHE, data.connectedActor)
return handleRequest(request, cacheFromEnv(env), data.connectedActor)
}
export async function handleRequest(request: Request, cache: KVNamespace, actor: Actor): Promise<Response> {
export async function handleRequest(request: Request, cache: Cache, actor: Actor): Promise<Response> {
const url = new URL(request.url)
if (url.searchParams.has('max_id')) {
// We just return the pregenerated notifications, without any filter for
@ -21,9 +23,9 @@ export async function handleRequest(request: Request, cache: KVNamespace, actor:
return new Response(JSON.stringify([]), { headers })
}
const timeline = await cache.get(actor.id + '/timeline/home')
const timeline = await cache.get<any>(actor.id + '/timeline/home')
if (timeline === null) {
return new Response(JSON.stringify([]), { headers })
}
return new Response(timeline, { headers })
return new Response(JSON.stringify(timeline), { headers })
}

Wyświetl plik

@ -27,6 +27,11 @@ variable "d1_id" {
sensitive = true
}
variable "do_cache_id" {
type = string
sensitive = true
}
variable "access_auth_domain" {
type = string
sensitive = true
@ -76,9 +81,11 @@ provider "cloudflare" {
api_token = var.cloudflare_api_token
}
resource "cloudflare_workers_kv_namespace" "wildebeest_cache" {
account_id = var.cloudflare_account_id
title = "wildebeest-${lower(var.gh_username)}-cache"
// The KV cache namespace isn't used anymore but Terraform isn't able
// to remove the binding from the Pages project, so leaving for now.
resource "cloudflare_workers_kv_namespace" "wildebeest_cache" {
account_id = var.cloudflare_account_id
title = "wildebeest-${lower(var.gh_username)}-cache"
}
resource "cloudflare_workers_kv_namespace" "terraform_state" {
@ -117,13 +124,19 @@ resource "cloudflare_pages_project" "wildebeest_pages_project" {
SENTRY_ACCESS_CLIENT_ID = var.sentry_access_client_id
SENTRY_ACCESS_CLIENT_SECRET = var.sentry_access_client_secret
}
kv_namespaces = {
KV_CACHE = sensitive(cloudflare_workers_kv_namespace.wildebeest_cache.id)
kv_namespaces = {
KV_CACHE = sensitive(cloudflare_workers_kv_namespace.wildebeest_cache.id)
}
d1_databases = {
DATABASE = sensitive(var.d1_id)
}
durable_object_namespaces = {
DO_CACHE = sensitive(var.do_cache_id)
}
compatibility_date = "2023-01-09"
}
}