kopia lustrzana https://github.com/cloudflare/wildebeest
MOW-118: switch from KV to DO for caching
rodzic
346b0a19dc
commit
9c11b74c2e
|
@ -166,6 +166,20 @@ jobs:
|
|||
echo "VAPID keys generated"
|
||||
fi
|
||||
|
||||
- name: Publish DO
|
||||
uses: cloudflare/wrangler-action@2.0.0
|
||||
with:
|
||||
apiToken: ${{ secrets.CF_API_TOKEN }}
|
||||
command: publish --config do/wrangler.toml
|
||||
env:
|
||||
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
|
||||
|
||||
- name: Retrieve DO namespace
|
||||
run: |
|
||||
curl https://api.cloudflare.com/client/v4/accounts/${{ secrets.CF_ACCOUNT_ID }}/workers/durable_objects/namespaces \
|
||||
-H 'Authorization: Bearer ${{ secrets.CF_API_TOKEN }}' \
|
||||
| jq -r '.result[] | select( .script == "wildebeest-do" ) | .id' | awk '{print "do_cache_id="$1}' >> $GITHUB_ENV
|
||||
|
||||
- name: Configure
|
||||
run: terraform plan && terraform apply -auto-approve
|
||||
continue-on-error: true
|
||||
|
@ -177,6 +191,7 @@ jobs:
|
|||
TF_VAR_cloudflare_deploy_domain: ${{ vars.CF_DEPLOY_DOMAIN }}
|
||||
TF_VAR_gh_username: ${{ env.OWNER_LOWER }}
|
||||
TF_VAR_d1_id: ${{ env.d1_id }}
|
||||
TF_VAR_do_cache_id: ${{ env.do_cache_id }}
|
||||
TF_VAR_access_auth_domain: ${{ env.auth_domain }}
|
||||
TF_VAR_wd_instance_title: ${{ vars.INSTANCE_TITLE }}
|
||||
TF_VAR_wd_admin_email: ${{ vars.ADMIN_EMAIL }}
|
||||
|
@ -214,18 +229,6 @@ jobs:
|
|||
env:
|
||||
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
|
||||
|
||||
- name: retrieve Wildebeest cache KV namespace
|
||||
uses: cloudflare/wrangler-action@2.0.0
|
||||
with:
|
||||
command: kv:namespace list | jq -r '.[] | select( .title == "wildebeest-${{ env.OWNER_LOWER }}-cache" ) | .id' | awk '{print "cache_kv="$1}' >> $GITHUB_ENV
|
||||
apiToken: ${{ secrets.CF_API_TOKEN }}
|
||||
preCommands: |
|
||||
echo "*** pre commands ***"
|
||||
apt-get update && apt-get -y install jq
|
||||
echo "******"
|
||||
env:
|
||||
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
|
||||
|
||||
- name: Create Queue
|
||||
uses: cloudflare/wrangler-action@2.0.0
|
||||
with:
|
||||
|
@ -243,9 +246,10 @@ jobs:
|
|||
echo "*** pre commands ***"
|
||||
echo -e "[[d1_databases]]\nbinding=\"DATABASE\"\ndatabase_name=\"wildebeest-${{ env.OWNER_LOWER }}\"\ndatabase_id=\"${{ env.d1_id }}\"\n" >> consumer/wrangler.toml
|
||||
|
||||
echo -e "[[kv_namespaces]]\n" >> consumer/wrangler.toml
|
||||
echo -e "binding=\"KV_CACHE\"\n" >> consumer/wrangler.toml
|
||||
echo -e "id=\"${{ env.cache_kv }}\"\n" >> consumer/wrangler.toml
|
||||
echo -e "[durable_objects]\n" >> consumer/wrangler.toml
|
||||
echo -e "bindings=[" >> consumer/wrangler.toml
|
||||
echo -e "{name=\"DO_CACHE\",class_name=\"WildebeestCache\",script_name=\"wildebeest-do\"}," >> consumer/wrangler.toml
|
||||
echo -e "]" >> consumer/wrangler.toml
|
||||
|
||||
echo -e "[vars]\n" >> consumer/wrangler.toml
|
||||
echo -e "DOMAIN=\"${{ vars.CF_DEPLOY_DOMAIN }}\"\n" >> consumer/wrangler.toml
|
||||
|
|
|
@ -0,0 +1,36 @@
|
|||
const CACHE_DO_NAME = 'cachev1'
|
||||
|
||||
export interface Cache {
|
||||
get<T>(key: string): Promise<T | null>
|
||||
put<T>(key: string, value: T): Promise<void>
|
||||
}
|
||||
|
||||
export function cacheFromEnv(env: any): Cache {
|
||||
return {
|
||||
async get<T>(key: string): Promise<T | null> {
|
||||
const id = env.DO_CACHE.idFromName(CACHE_DO_NAME)
|
||||
const stub = env.DO_CACHE.get(id)
|
||||
|
||||
const res = await stub.fetch('http://cache/' + key)
|
||||
if (!res.ok) {
|
||||
throw new Error(`DO cache returned ${res.status}: ${await res.text()}`)
|
||||
}
|
||||
|
||||
return (await res.json()) as T
|
||||
},
|
||||
|
||||
async put<T>(key: string, value: T): Promise<void> {
|
||||
const id = env.DO_CACHE.idFromName(CACHE_DO_NAME)
|
||||
const stub = env.DO_CACHE.get(id)
|
||||
|
||||
const res = await stub.fetch('http://cache/', {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify({ key, value }),
|
||||
})
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`DO cache returned ${res.status}: ${await res.text()}`)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
|
@ -10,6 +10,7 @@ import { WebPushResult } from 'wildebeest/backend/src/webpush/webpushinfos'
|
|||
import type { Actor } from 'wildebeest/backend/src/activitypub/actors'
|
||||
import type { NotificationType, Notification } from 'wildebeest/backend/src/types/notification'
|
||||
import { getSubscriptionForAllClients } from 'wildebeest/backend/src/mastodon/subscription'
|
||||
import type { Cache } from 'wildebeest/backend/src/cache'
|
||||
|
||||
export async function createNotification(
|
||||
db: D1Database,
|
||||
|
@ -250,7 +251,7 @@ export async function getNotifications(db: D1Database, actor: Actor, domain: str
|
|||
return out
|
||||
}
|
||||
|
||||
export async function pregenerateNotifications(db: D1Database, cache: KVNamespace, actor: Actor, domain: string) {
|
||||
export async function pregenerateNotifications(db: D1Database, cache: Cache, actor: Actor, domain: string) {
|
||||
const notifications = await getNotifications(db, actor, domain)
|
||||
await cache.put(actor.id + '/notifications', JSON.stringify(notifications))
|
||||
await cache.put(actor.id + '/notifications', notifications)
|
||||
}
|
||||
|
|
|
@ -3,10 +3,11 @@ import { getFollowingId } from 'wildebeest/backend/src/mastodon/follow'
|
|||
import type { Actor } from 'wildebeest/backend/src/activitypub/actors/'
|
||||
import { toMastodonStatusFromRow } from './status'
|
||||
import { PUBLIC_GROUP } from 'wildebeest/backend/src/activitypub/activities'
|
||||
import type { Cache } from 'wildebeest/backend/src/cache'
|
||||
|
||||
export async function pregenerateTimelines(domain: string, db: D1Database, cache: KVNamespace, actor: Actor) {
|
||||
export async function pregenerateTimelines(domain: string, db: D1Database, cache: Cache, actor: Actor) {
|
||||
const timeline = await getHomeTimeline(domain, db, actor)
|
||||
await cache.put(actor.id + '/timeline/home', JSON.stringify(timeline))
|
||||
await cache.put(actor.id + '/timeline/home', timeline)
|
||||
}
|
||||
|
||||
export async function getHomeTimeline(domain: string, db: D1Database, actor: Actor): Promise<Array<MastodonStatus>> {
|
||||
|
|
|
@ -2,9 +2,10 @@ import type { Queue, MessageBody } from 'wildebeest/backend/src/types/queue'
|
|||
|
||||
export interface Env {
|
||||
DATABASE: D1Database
|
||||
KV_CACHE: KVNamespace
|
||||
// FIXME: shouldn't it be USER_KEY?
|
||||
userKEK: string
|
||||
QUEUE: Queue<MessageBody>
|
||||
DO_CACHE: DurableObjectNamespace
|
||||
|
||||
CF_ACCOUNT_ID: string
|
||||
CF_API_TOKEN: string
|
||||
|
|
|
@ -4,7 +4,7 @@ import { createPublicNote } from 'wildebeest/backend/src/activitypub/objects/not
|
|||
import { createNotification, insertFollowNotification } from 'wildebeest/backend/src/mastodon/notification'
|
||||
import { createPerson } from 'wildebeest/backend/src/activitypub/actors'
|
||||
import * as notifications from 'wildebeest/functions/api/v1/notifications'
|
||||
import { makeDB, assertJSON, createTestClient } from '../utils'
|
||||
import { makeCache, makeDB, assertJSON, createTestClient } from '../utils'
|
||||
import { strict as assert } from 'node:assert/strict'
|
||||
import { sendLikeNotification } from 'wildebeest/backend/src/mastodon/notification'
|
||||
import { createSubscription } from 'wildebeest/backend/src/mastodon/subscription'
|
||||
|
@ -32,15 +32,13 @@ describe('Mastodon APIs', () => {
|
|||
test('returns notifications stored in KV cache', async () => {
|
||||
const db = await makeDB()
|
||||
const connectedActor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com')
|
||||
const kv_cache: any = {
|
||||
async get(key: string) {
|
||||
assert.equal(key, connectedActor.id + '/notifications')
|
||||
return 'cached data'
|
||||
},
|
||||
}
|
||||
const cache = makeCache()
|
||||
|
||||
await cache.put(connectedActor.id + '/notifications', 12345)
|
||||
|
||||
const req = new Request('https://' + domain)
|
||||
const data = await notifications.handleRequest(req, kv_cache, connectedActor)
|
||||
assert.equal(await data.text(), 'cached data')
|
||||
const data = await notifications.handleRequest(req, cache, connectedActor)
|
||||
assert.equal(await data.json(), 12345)
|
||||
})
|
||||
|
||||
test('returns notifications stored in db', async () => {
|
||||
|
|
|
@ -12,7 +12,7 @@ import * as statuses_context from 'wildebeest/functions/api/v1/statuses/[id]/con
|
|||
import { createPerson } from 'wildebeest/backend/src/activitypub/actors'
|
||||
import { insertLike } from 'wildebeest/backend/src/mastodon/like'
|
||||
import { insertReblog } from 'wildebeest/backend/src/mastodon/reblog'
|
||||
import { isUrlValid, makeDB, assertJSON, streamToArrayBuffer, makeQueue } from '../utils'
|
||||
import { isUrlValid, makeDB, assertJSON, streamToArrayBuffer, makeQueue, makeCache } from '../utils'
|
||||
import * as activities from 'wildebeest/backend/src/activitypub/activities'
|
||||
import { addFollowing, acceptFollowing } from 'wildebeest/backend/src/mastodon/follow'
|
||||
import { MessageType } from 'wildebeest/backend/src/types/queue'
|
||||
|
@ -20,9 +20,7 @@ import { MessageType } from 'wildebeest/backend/src/types/queue'
|
|||
const userKEK = 'test_kek4'
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
|
||||
const domain = 'cloudflare.com'
|
||||
const kv_cache: any = {
|
||||
async put() {},
|
||||
}
|
||||
const cache = makeCache()
|
||||
|
||||
describe('Mastodon APIs', () => {
|
||||
describe('statuses', () => {
|
||||
|
@ -38,7 +36,7 @@ describe('Mastodon APIs', () => {
|
|||
})
|
||||
|
||||
const connectedActor: any = {}
|
||||
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache)
|
||||
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache)
|
||||
assert.equal(res.status, 400)
|
||||
})
|
||||
|
||||
|
@ -58,7 +56,7 @@ describe('Mastodon APIs', () => {
|
|||
})
|
||||
|
||||
const connectedActor = actor
|
||||
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache)
|
||||
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache)
|
||||
assert.equal(res.status, 200)
|
||||
assertJSON(res)
|
||||
|
||||
|
@ -96,14 +94,7 @@ describe('Mastodon APIs', () => {
|
|||
const db = await makeDB()
|
||||
const queue = makeQueue()
|
||||
const actor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com')
|
||||
|
||||
let cache = null
|
||||
const kv_cache: any = {
|
||||
async put(key: string, value: any) {
|
||||
assert.equal(key, actor.id + '/timeline/home')
|
||||
cache = value
|
||||
},
|
||||
}
|
||||
const cache = makeCache()
|
||||
|
||||
const body = {
|
||||
status: 'my status',
|
||||
|
@ -115,16 +106,17 @@ describe('Mastodon APIs', () => {
|
|||
body: JSON.stringify(body),
|
||||
})
|
||||
|
||||
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache)
|
||||
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache)
|
||||
assert.equal(res.status, 200)
|
||||
assertJSON(res)
|
||||
|
||||
const data = await res.json<any>()
|
||||
|
||||
assert(cache)
|
||||
const timeline = JSON.parse(cache)
|
||||
assert.equal(timeline.length, 1)
|
||||
assert.equal(timeline[0].id, data.id)
|
||||
const cachedData = await cache.get<any>(actor.id + '/timeline/home')
|
||||
console.log({ cachedData })
|
||||
assert(cachedData)
|
||||
assert.equal(cachedData.length, 1)
|
||||
assert.equal(cachedData[0].id, data.id)
|
||||
})
|
||||
|
||||
test("create new status adds to Actor's outbox", async () => {
|
||||
|
@ -143,7 +135,7 @@ describe('Mastodon APIs', () => {
|
|||
})
|
||||
|
||||
const connectedActor = actor
|
||||
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache)
|
||||
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache)
|
||||
assert.equal(res.status, 200)
|
||||
|
||||
const row = await db.prepare(`SELECT count(*) as count FROM outbox_objects`).first()
|
||||
|
@ -171,7 +163,7 @@ describe('Mastodon APIs', () => {
|
|||
body: JSON.stringify(body),
|
||||
})
|
||||
|
||||
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache)
|
||||
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache)
|
||||
assert.equal(res.status, 200)
|
||||
|
||||
assert.equal(queue.messages.length, 2)
|
||||
|
@ -250,7 +242,7 @@ describe('Mastodon APIs', () => {
|
|||
})
|
||||
|
||||
const connectedActor = actor
|
||||
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache)
|
||||
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache)
|
||||
assert.equal(res.status, 200)
|
||||
|
||||
assert(deliveredNote)
|
||||
|
@ -281,7 +273,7 @@ describe('Mastodon APIs', () => {
|
|||
body: JSON.stringify(body),
|
||||
})
|
||||
|
||||
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache)
|
||||
const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache)
|
||||
assert.equal(res.status, 200)
|
||||
|
||||
const data = await res.json<any>()
|
||||
|
@ -560,7 +552,7 @@ describe('Mastodon APIs', () => {
|
|||
body: JSON.stringify(body),
|
||||
})
|
||||
|
||||
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache)
|
||||
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache)
|
||||
assert.equal(res.status, 404)
|
||||
})
|
||||
|
||||
|
@ -581,7 +573,7 @@ describe('Mastodon APIs', () => {
|
|||
body: JSON.stringify(body),
|
||||
})
|
||||
|
||||
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache)
|
||||
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache)
|
||||
assert.equal(res.status, 200)
|
||||
|
||||
const data = await res.json<any>()
|
||||
|
@ -625,7 +617,7 @@ describe('Mastodon APIs', () => {
|
|||
body: JSON.stringify(body),
|
||||
})
|
||||
|
||||
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache)
|
||||
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache)
|
||||
assert.equal(res.status, 400)
|
||||
const data = await res.json<any>()
|
||||
assert(data.error.includes('Limit exceeded'))
|
||||
|
@ -650,7 +642,7 @@ describe('Mastodon APIs', () => {
|
|||
body,
|
||||
})
|
||||
|
||||
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache)
|
||||
const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache)
|
||||
assert.equal(res.status, 400)
|
||||
const data = await res.json<any>()
|
||||
assert(data.error.includes('Limit exceeded'))
|
||||
|
|
|
@ -5,7 +5,7 @@ import { addFollowing, acceptFollowing } from 'wildebeest/backend/src/mastodon/f
|
|||
import { createPublicNote, createPrivateNote } from 'wildebeest/backend/src/activitypub/objects/note'
|
||||
import { addObjectInOutbox } from 'wildebeest/backend/src/activitypub/actors/outbox'
|
||||
import { createPerson } from 'wildebeest/backend/src/activitypub/actors'
|
||||
import { makeDB, assertCORS, assertJSON } from '../utils'
|
||||
import { makeDB, assertCORS, assertJSON, makeCache } from '../utils'
|
||||
import * as timelines_home from 'wildebeest/functions/api/v1/timelines/home'
|
||||
import * as timelines_public from 'wildebeest/functions/api/v1/timelines/public'
|
||||
import * as timelines from 'wildebeest/backend/src/mastodon/timeline'
|
||||
|
@ -106,27 +106,20 @@ describe('Mastodon APIs', () => {
|
|||
test('home returns cache', async () => {
|
||||
const db = await makeDB()
|
||||
const connectedActor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com')
|
||||
const kv_cache: any = {
|
||||
async get(key: string) {
|
||||
assert.equal(key, connectedActor.id + '/timeline/home')
|
||||
return 'cached data'
|
||||
},
|
||||
}
|
||||
const cache = makeCache()
|
||||
await cache.put(connectedActor.id + '/timeline/home', 12345)
|
||||
|
||||
const req = new Request('https://' + domain)
|
||||
const data = await timelines_home.handleRequest(req, kv_cache, connectedActor)
|
||||
assert.equal(await data.text(), 'cached data')
|
||||
const data = await timelines_home.handleRequest(req, cache, connectedActor)
|
||||
assert.equal(await data.json(), 12345)
|
||||
})
|
||||
|
||||
test('home returns empty if not in cache', async () => {
|
||||
const db = await makeDB()
|
||||
const connectedActor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com')
|
||||
const kv_cache: any = {
|
||||
async get() {
|
||||
return null
|
||||
},
|
||||
}
|
||||
const cache = makeCache()
|
||||
const req = new Request('https://' + domain)
|
||||
const data = await timelines_home.handleRequest(req, kv_cache, connectedActor)
|
||||
const data = await timelines_home.handleRequest(req, cache, connectedActor)
|
||||
const posts = await data.json<Array<any>>()
|
||||
|
||||
assert.equal(posts.length, 0)
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import { strict as assert } from 'node:assert/strict'
|
||||
import type { Cache } from 'wildebeest/backend/src/cache'
|
||||
import type { Queue } from 'wildebeest/backend/src/types/queue'
|
||||
import { createClient } from 'wildebeest/backend/src/mastodon/client'
|
||||
import type { Client } from 'wildebeest/backend/src/mastodon/client'
|
||||
|
@ -91,6 +92,24 @@ export function makeQueue(): TestQueue {
|
|||
}
|
||||
}
|
||||
|
||||
export function makeCache(): Cache {
|
||||
const cache: any = {}
|
||||
|
||||
return {
|
||||
async get<T>(key: string): Promise<T | null> {
|
||||
if (cache[key]) {
|
||||
return cache[key] as T
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
async put<T>(key: string, value: T): Promise<void> {
|
||||
cache[key] = value
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export function isUUID(v: string): boolean {
|
||||
assert.equal(typeof v, 'string')
|
||||
if (v.match('^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$') === null) {
|
||||
|
|
|
@ -2,6 +2,7 @@ import type { MessageBody, InboxMessageBody } from 'wildebeest/backend/src/types
|
|||
import * as activityHandler from 'wildebeest/backend/src/activitypub/activities/handle'
|
||||
import * as notification from 'wildebeest/backend/src/mastodon/notification'
|
||||
import * as timeline from 'wildebeest/backend/src/mastodon/timeline'
|
||||
import { cacheFromEnv } from 'wildebeest/backend/src/cache'
|
||||
import type { Actor } from 'wildebeest/backend/src/activitypub/actors'
|
||||
import type { Env } from './'
|
||||
|
||||
|
@ -9,7 +10,7 @@ export async function handleInboxMessage(env: Env, actor: Actor, message: InboxM
|
|||
const domain = env.DOMAIN
|
||||
const db = env.DATABASE
|
||||
const adminEmail = env.ADMIN_EMAIL
|
||||
const cache = env.KV_CACHE
|
||||
const cache = cacheFromEnv(env)
|
||||
const activity = message.activity
|
||||
|
||||
await activityHandler.handle(domain, activity, db, message.userKEK, adminEmail, message.vapidKeys)
|
||||
|
|
|
@ -12,7 +12,7 @@ export type Env = {
|
|||
DATABASE: D1Database
|
||||
DOMAIN: string
|
||||
ADMIN_EMAIL: string
|
||||
KV_CACHE: KVNamespace
|
||||
DO_CACHE: DurableObjectNamespace
|
||||
}
|
||||
|
||||
export default {
|
||||
|
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"name": "wildebeest-do",
|
||||
"version": "0.0.0",
|
||||
"devDependencies": {
|
||||
"@cloudflare/workers-types": "^4.20221111.1",
|
||||
"typescript": "^4.9.4",
|
||||
"wrangler": "2.7.1"
|
||||
},
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "wrangler dev",
|
||||
"deploy": "wrangler publish"
|
||||
}
|
||||
}
|
|
@ -0,0 +1,49 @@
|
|||
export interface Env {
|
||||
DO: DurableObjectNamespace
|
||||
}
|
||||
|
||||
export default {
|
||||
async fetch(request: Request, env: Env) {
|
||||
try {
|
||||
const id = env.DO.idFromName('default')
|
||||
const obj = env.DO.get(id)
|
||||
return obj.fetch(request)
|
||||
} catch (err: any) {
|
||||
return new Response(err.stack, { status: 500 })
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
export class WildebeestCache {
|
||||
private storage: DurableObjectStorage
|
||||
|
||||
constructor(state: DurableObjectState) {
|
||||
this.storage = state.storage
|
||||
}
|
||||
|
||||
async fetch(request: Request) {
|
||||
if (request.method === 'GET') {
|
||||
const { pathname } = new URL(request.url)
|
||||
const key = pathname.slice(1) // remove the leading slash from path
|
||||
|
||||
const value = await this.storage.get(key)
|
||||
if (value === undefined) {
|
||||
console.log(`Get ${key} MISS`)
|
||||
return new Response('', { status: 404 })
|
||||
}
|
||||
|
||||
console.log(`Get ${key} HIT`)
|
||||
return new Response(JSON.stringify(value))
|
||||
}
|
||||
|
||||
if (request.method === 'PUT') {
|
||||
const { key, value } = await request.json<any>()
|
||||
console.log(`Set ${key}`)
|
||||
|
||||
await this.storage.put(key, value)
|
||||
return new Response('', { status: 201 })
|
||||
}
|
||||
|
||||
return new Response('', { status: 400 })
|
||||
}
|
||||
}
|
|
@ -0,0 +1,106 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
/* Visit https://aka.ms/tsconfig.json to read more about this file */
|
||||
|
||||
/* Projects */
|
||||
// "incremental": true, /* Enable incremental compilation */
|
||||
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
|
||||
// "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */
|
||||
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */
|
||||
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
|
||||
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
|
||||
|
||||
/* Language and Environment */
|
||||
"target": "es2021" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
|
||||
"lib": [
|
||||
"es2021"
|
||||
] /* Specify a set of bundled library declaration files that describe the target runtime environment. */,
|
||||
"jsx": "react" /* Specify what JSX code is generated. */,
|
||||
// "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */
|
||||
// "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */
|
||||
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */
|
||||
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
|
||||
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */
|
||||
// "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */
|
||||
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
|
||||
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
|
||||
|
||||
/* Modules */
|
||||
"module": "es2022" /* Specify what module code is generated. */,
|
||||
// "rootDir": "./", /* Specify the root folder within your source files. */
|
||||
"moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */,
|
||||
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
|
||||
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
|
||||
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
|
||||
// "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */
|
||||
"types": [
|
||||
"@cloudflare/workers-types",
|
||||
"jest"
|
||||
] /* Specify type package names to be included without being referenced in a source file. */,
|
||||
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
|
||||
"resolveJsonModule": true /* Enable importing .json files */,
|
||||
// "noResolve": true, /* Disallow `import`s, `require`s or `<reference>`s from expanding the number of files TypeScript should add to a project. */
|
||||
|
||||
/* JavaScript Support */
|
||||
"allowJs": true /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */,
|
||||
"checkJs": false /* Enable error reporting in type-checked JavaScript files. */,
|
||||
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */
|
||||
|
||||
/* Emit */
|
||||
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
|
||||
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
|
||||
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
|
||||
// "sourceMap": true, /* Create source map files for emitted JavaScript files. */
|
||||
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */
|
||||
// "outDir": "./", /* Specify an output folder for all emitted files. */
|
||||
// "removeComments": true, /* Disable emitting comments. */
|
||||
"noEmit": true /* Disable emitting files from a compilation. */,
|
||||
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
|
||||
// "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */
|
||||
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
|
||||
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
|
||||
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
|
||||
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
|
||||
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
|
||||
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
|
||||
// "newLine": "crlf", /* Set the newline character for emitting files. */
|
||||
// "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */
|
||||
// "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */
|
||||
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
|
||||
// "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */
|
||||
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
|
||||
// "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
|
||||
|
||||
/* Interop Constraints */
|
||||
"isolatedModules": true /* Ensure that each file can be safely transpiled without relying on other imports. */,
|
||||
"allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */,
|
||||
// "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */,
|
||||
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
|
||||
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
|
||||
|
||||
/* Type Checking */
|
||||
"strict": true /* Enable all strict type-checking options. */,
|
||||
// "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */
|
||||
// "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */
|
||||
// "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
|
||||
// "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */
|
||||
// "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */
|
||||
// "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */
|
||||
// "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */
|
||||
// "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */
|
||||
// "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */
|
||||
// "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */
|
||||
// "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
|
||||
// "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
|
||||
// "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
|
||||
// "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */
|
||||
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
|
||||
// "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */
|
||||
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
|
||||
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
|
||||
|
||||
/* Completeness */
|
||||
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
|
||||
"skipLibCheck": true /* Skip type checking all .d.ts files. */
|
||||
}
|
||||
}
|
|
@ -0,0 +1,7 @@
|
|||
name = "wildebeest-do"
|
||||
main = "src/index.ts"
|
||||
compatibility_date = "2023-01-18"
|
||||
|
||||
[[migrations]]
|
||||
tag = "v1"
|
||||
new_classes = ["WildebeestCache"]
|
|
@ -6,14 +6,15 @@ import type { Account, MastodonStatus } from 'wildebeest/frontend/src/types'
|
|||
|
||||
const kek = 'test-kek'
|
||||
/* eslint-disable @typescript-eslint/no-empty-function */
|
||||
const queue = {
|
||||
async send() {},
|
||||
async sendBatch() {},
|
||||
const queue: unknown = {
|
||||
send() {},
|
||||
sendBatch() {},
|
||||
}
|
||||
const kv_cache = {
|
||||
async put() {},
|
||||
/* eslint-disable @typescript-eslint/no-empty-function */
|
||||
const cache: unknown = {
|
||||
get() {},
|
||||
put() {},
|
||||
}
|
||||
/* eslint-enable @typescript-eslint/no-empty-function */
|
||||
|
||||
/**
|
||||
* Run helper commands to initialize the database with actors, statuses, etc.
|
||||
|
@ -30,7 +31,7 @@ export async function init(domain: string, db: D1Database) {
|
|||
const reblogger = await getOrCreatePerson(domain, db, rebloggerAccount)
|
||||
// Reblog an arbitrary status with this reblogger
|
||||
const statusToReblog = loadedStatuses[2]
|
||||
await reblogStatus(db, reblogger, statusToReblog)
|
||||
await reblogStatus(db, reblogger, statusToReblog, domain)
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -49,7 +50,7 @@ async function createStatus(db: D1Database, actor: Person, status: string, visib
|
|||
headers,
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
const resp = await statusesAPI.handleRequest(req, db, actor, kek, queue, kv_cache as unknown as KVNamespace)
|
||||
const resp = await statusesAPI.handleRequest(req, db, actor, kek, queue, cache)
|
||||
return (await resp.json()) as MastodonStatus
|
||||
}
|
||||
|
||||
|
@ -70,6 +71,6 @@ async function getOrCreatePerson(
|
|||
return newPerson
|
||||
}
|
||||
|
||||
async function reblogStatus(db: D1Database, actor: Person, status: MastodonStatus) {
|
||||
await reblogAPI.handleRequest(db, status.id, actor, kek, queue)
|
||||
async function reblogStatus(db: D1Database, actor: Person, status: MastodonStatus, domain: string) {
|
||||
await reblogAPI.handleRequest(db, status.id, actor, kek, queue, domain)
|
||||
}
|
||||
|
|
|
@ -4,9 +4,11 @@ import { cors } from 'wildebeest/backend/src/utils/cors'
|
|||
import type { Env } from 'wildebeest/backend/src/types/env'
|
||||
import type { Person } from 'wildebeest/backend/src/activitypub/actors'
|
||||
import type { ContextData } from 'wildebeest/backend/src/types/context'
|
||||
import type { Cache } from 'wildebeest/backend/src/cache'
|
||||
import { cacheFromEnv } from 'wildebeest/backend/src/cache'
|
||||
|
||||
export const onRequest: PagesFunction<Env, any, ContextData> = async ({ request, env, data }) => {
|
||||
return handleRequest(request, env.KV_CACHE, data.connectedActor)
|
||||
return handleRequest(request, cacheFromEnv(env), data.connectedActor)
|
||||
}
|
||||
|
||||
const headers = {
|
||||
|
@ -14,7 +16,7 @@ const headers = {
|
|||
'content-type': 'application/json; charset=utf-8',
|
||||
}
|
||||
|
||||
export async function handleRequest(request: Request, cache: KVNamespace, connectedActor: Person): Promise<Response> {
|
||||
export async function handleRequest(request: Request, cache: Cache, connectedActor: Person): Promise<Response> {
|
||||
const url = new URL(request.url)
|
||||
if (url.searchParams.has('max_id')) {
|
||||
// We just return the pregenerated notifications, without any filter for
|
||||
|
@ -23,9 +25,9 @@ export async function handleRequest(request: Request, cache: KVNamespace, connec
|
|||
return new Response(JSON.stringify([]), { headers })
|
||||
}
|
||||
|
||||
const notifications = await cache.get(connectedActor.id + '/notifications')
|
||||
const notifications = await cache.get<any>(connectedActor.id + '/notifications')
|
||||
if (notifications === null) {
|
||||
return new Response(JSON.stringify([]), { headers })
|
||||
}
|
||||
return new Response(notifications, { headers })
|
||||
return new Response(JSON.stringify(notifications), { headers })
|
||||
}
|
||||
|
|
|
@ -21,6 +21,8 @@ import { readBody } from 'wildebeest/backend/src/utils/body'
|
|||
import * as errors from 'wildebeest/backend/src/errors'
|
||||
import type { Visibility } from 'wildebeest/backend/src/types'
|
||||
import { toMastodonStatusFromObject } from 'wildebeest/backend/src/mastodon/status'
|
||||
import type { Cache } from 'wildebeest/backend/src/cache'
|
||||
import { cacheFromEnv } from 'wildebeest/backend/src/cache'
|
||||
|
||||
type StatusCreate = {
|
||||
status: string
|
||||
|
@ -31,7 +33,7 @@ type StatusCreate = {
|
|||
}
|
||||
|
||||
export const onRequest: PagesFunction<Env, any, ContextData> = async ({ request, env, data }) => {
|
||||
return handleRequest(request, env.DATABASE, data.connectedActor, env.userKEK, env.QUEUE, env.KV_CACHE)
|
||||
return handleRequest(request, env.DATABASE, data.connectedActor, env.userKEK, env.QUEUE, cacheFromEnv(env))
|
||||
}
|
||||
|
||||
// FIXME: add tests for delivery to followers and mentions to a specific Actor.
|
||||
|
@ -41,7 +43,7 @@ export async function handleRequest(
|
|||
connectedActor: Person,
|
||||
userKEK: string,
|
||||
queue: Queue<DeliverMessageBody>,
|
||||
cache: KVNamespace
|
||||
cache: Cache
|
||||
): Promise<Response> {
|
||||
// TODO: implement Idempotency-Key
|
||||
|
||||
|
|
|
@ -2,6 +2,8 @@ import type { Env } from 'wildebeest/backend/src/types/env'
|
|||
import { cors } from 'wildebeest/backend/src/utils/cors'
|
||||
import type { ContextData } from 'wildebeest/backend/src/types/context'
|
||||
import type { Actor } from 'wildebeest/backend/src/activitypub/actors/'
|
||||
import type { Cache } from 'wildebeest/backend/src/cache'
|
||||
import { cacheFromEnv } from 'wildebeest/backend/src/cache'
|
||||
|
||||
const headers = {
|
||||
...cors(),
|
||||
|
@ -9,10 +11,10 @@ const headers = {
|
|||
}
|
||||
|
||||
export const onRequest: PagesFunction<Env, any, ContextData> = async ({ request, env, data }) => {
|
||||
return handleRequest(request, env.KV_CACHE, data.connectedActor)
|
||||
return handleRequest(request, cacheFromEnv(env), data.connectedActor)
|
||||
}
|
||||
|
||||
export async function handleRequest(request: Request, cache: KVNamespace, actor: Actor): Promise<Response> {
|
||||
export async function handleRequest(request: Request, cache: Cache, actor: Actor): Promise<Response> {
|
||||
const url = new URL(request.url)
|
||||
if (url.searchParams.has('max_id')) {
|
||||
// We just return the pregenerated notifications, without any filter for
|
||||
|
@ -21,9 +23,9 @@ export async function handleRequest(request: Request, cache: KVNamespace, actor:
|
|||
return new Response(JSON.stringify([]), { headers })
|
||||
}
|
||||
|
||||
const timeline = await cache.get(actor.id + '/timeline/home')
|
||||
const timeline = await cache.get<any>(actor.id + '/timeline/home')
|
||||
if (timeline === null) {
|
||||
return new Response(JSON.stringify([]), { headers })
|
||||
}
|
||||
return new Response(timeline, { headers })
|
||||
return new Response(JSON.stringify(timeline), { headers })
|
||||
}
|
||||
|
|
13
tf/main.tf
13
tf/main.tf
|
@ -27,6 +27,11 @@ variable "d1_id" {
|
|||
sensitive = true
|
||||
}
|
||||
|
||||
variable "do_cache_id" {
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "access_auth_domain" {
|
||||
type = string
|
||||
sensitive = true
|
||||
|
@ -76,6 +81,8 @@ provider "cloudflare" {
|
|||
api_token = var.cloudflare_api_token
|
||||
}
|
||||
|
||||
// The KV cache namespace isn't used anymore but Terraform isn't able
|
||||
// to remove the binding from the Pages project, so leaving for now.
|
||||
resource "cloudflare_workers_kv_namespace" "wildebeest_cache" {
|
||||
account_id = var.cloudflare_account_id
|
||||
title = "wildebeest-${lower(var.gh_username)}-cache"
|
||||
|
@ -117,13 +124,19 @@ resource "cloudflare_pages_project" "wildebeest_pages_project" {
|
|||
SENTRY_ACCESS_CLIENT_ID = var.sentry_access_client_id
|
||||
SENTRY_ACCESS_CLIENT_SECRET = var.sentry_access_client_secret
|
||||
}
|
||||
|
||||
kv_namespaces = {
|
||||
KV_CACHE = sensitive(cloudflare_workers_kv_namespace.wildebeest_cache.id)
|
||||
}
|
||||
|
||||
d1_databases = {
|
||||
DATABASE = sensitive(var.d1_id)
|
||||
}
|
||||
|
||||
durable_object_namespaces = {
|
||||
DO_CACHE = sensitive(var.do_cache_id)
|
||||
}
|
||||
|
||||
compatibility_date = "2023-01-09"
|
||||
}
|
||||
}
|
||||
|
|
Ładowanie…
Reference in New Issue