From e13c0fb670f198caefb90f790996e7f2211b5e75 Mon Sep 17 00:00:00 2001 From: Sven Sauleau Date: Tue, 7 Feb 2023 11:49:39 +0000 Subject: [PATCH] MOW-133: add idempotency for posting --- backend/src/mastodon/idempotency.ts | 51 ++++++++++++++++++++++++ backend/test/activitypub/follow.spec.ts | 1 - backend/test/mastodon/statuses.spec.ts | 40 ++++++++++++++++++- functions/api/v1/statuses.ts | 29 ++++++++++---- functions/api/v1/statuses/[id].ts | 1 + migrations/0005_add_idempotency_keys.sql | 9 +++++ 6 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 backend/src/mastodon/idempotency.ts create mode 100644 migrations/0005_add_idempotency_keys.sql diff --git a/backend/src/mastodon/idempotency.ts b/backend/src/mastodon/idempotency.ts new file mode 100644 index 0000000..efa59b7 --- /dev/null +++ b/backend/src/mastodon/idempotency.ts @@ -0,0 +1,51 @@ +import type { APObject } from 'wildebeest/backend/src/activitypub/objects' +import { + mastodonIdSymbol, + originalActorIdSymbol, + originalObjectIdSymbol, +} from 'wildebeest/backend/src/activitypub/objects' + +export async function insertKey(db: D1Database, key: string, obj: APObject): Promise { + const query = ` + INSERT INTO idempotency_keys (key, object_id, expires_at) + VALUES (?1, ?2, datetime('now', '+1 hour')) + ` + + const { success, error } = await db.prepare(query).bind(key, obj.id.toString()).run() + if (!success) { + throw new Error('SQL error: ' + error) + } +} + +export async function hasKey(db: D1Database, key: string): Promise { + const query = ` + SELECT objects.* + FROM idempotency_keys + INNER JOIN objects ON objects.id = idempotency_keys.object_id + WHERE idempotency_keys.key = ?1 AND expires_at >= datetime() + ` + + const { results, success, error } = await db.prepare(query).bind(key).all() + if (!success) { + throw new Error('SQL error: ' + error) + } + + if (!results || results.length === 0) { + return null + } + + const result = results[0] + const properties = JSON.parse(result.properties) + + return { + published: new Date(result.cdate).toISOString(), + ...properties, + + type: result.type, + id: new URL(result.id), + + [mastodonIdSymbol]: result.mastodon_id, + [originalActorIdSymbol]: result.original_actor_id, + [originalObjectIdSymbol]: result.original_object_id, + } as APObject +} diff --git a/backend/test/activitypub/follow.spec.ts b/backend/test/activitypub/follow.spec.ts index e133830..ccc804c 100644 --- a/backend/test/activitypub/follow.spec.ts +++ b/backend/test/activitypub/follow.spec.ts @@ -27,7 +27,6 @@ describe('ActivityPub', () => { assert.equal(request.method, 'POST') const data = await request.json() receivedActivity = data - console.log({ receivedActivity }) return new Response('') } diff --git a/backend/test/mastodon/statuses.spec.ts b/backend/test/mastodon/statuses.spec.ts index d95b34c..d626910 100644 --- a/backend/test/mastodon/statuses.spec.ts +++ b/backend/test/mastodon/statuses.spec.ts @@ -115,7 +115,6 @@ describe('Mastodon APIs', () => { const data = await res.json() const cachedData = await cache.get(actor.id + '/timeline/home') - console.log({ cachedData }) assert(cachedData) assert.equal(cachedData.length, 1) assert.equal(cachedData[0].id, data.id) @@ -875,5 +874,44 @@ describe('Mastodon APIs', () => { assert.equal(queue.messages[1].actorId, actor.id.toString()) assert.equal(queue.messages[1].toActorId, actor3.id.toString()) }) + + test('create duplicate statuses idempotency', async () => { + const db = await makeDB() + const queue = makeQueue() + const actor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com') + + const idempotencyKey = 'abcd' + + const body = { status: 'my status', visibility: 'public' } + const req = () => + new Request('https://example.com', { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'idempotency-key': idempotencyKey, + }, + body: JSON.stringify(body), + }) + + const res1 = await statuses.handleRequest(req(), db, actor, userKEK, queue, cache) + assert.equal(res1.status, 200) + const data1 = await res1.json() + + const res2 = await statuses.handleRequest(req(), db, actor, userKEK, queue, cache) + assert.equal(res2.status, 200) + const data2 = await res2.json() + + assert.deepEqual(data1, data2) + + { + const row = await db.prepare(`SELECT count(*) as count FROM objects`).first<{ count: number }>() + assert.equal(row.count, 1) + } + + { + const row = await db.prepare(`SELECT count(*) as count FROM idempotency_keys`).first<{ count: number }>() + assert.equal(row.count, 1) + } + }) }) }) diff --git a/functions/api/v1/statuses.ts b/functions/api/v1/statuses.ts index b7f95fd..2527ee3 100644 --- a/functions/api/v1/statuses.ts +++ b/functions/api/v1/statuses.ts @@ -1,5 +1,6 @@ // https://docs.joinmastodon.org/methods/statuses/#create +import type { Note } from 'wildebeest/backend/src/activitypub/objects/note' import { cors } from 'wildebeest/backend/src/utils/cors' import type { APObject } from 'wildebeest/backend/src/activitypub/objects' import { insertReply } from 'wildebeest/backend/src/mastodon/reply' @@ -21,6 +22,7 @@ import { toMastodonStatusFromObject } from 'wildebeest/backend/src/mastodon/stat import type { Cache } from 'wildebeest/backend/src/cache' import { cacheFromEnv } from 'wildebeest/backend/src/cache' import { enrichStatus } from 'wildebeest/backend/src/mastodon/microformats' +import * as idempotency from 'wildebeest/backend/src/mastodon/idempotency' import { newMention } from 'wildebeest/backend/src/activitypub/objects/mention' import { originalObjectIdSymbol } from 'wildebeest/backend/src/activitypub/objects' @@ -45,12 +47,26 @@ export async function handleRequest( queue: Queue, cache: Cache ): Promise { - // TODO: implement Idempotency-Key - if (request.method !== 'POST') { return new Response('', { status: 400 }) } + const domain = new URL(request.url).hostname + const headers = { + ...cors(), + 'content-type': 'application/json; charset=utf-8', + } + + const idempotencyKey = request.headers.get('Idempotency-Key') + + if (idempotencyKey !== null) { + const maybeObject = await idempotency.hasKey(db, idempotencyKey) + if (maybeObject !== null) { + const res = await toMastodonStatusFromObject(db, maybeObject as Note, domain) + return new Response(JSON.stringify(res), { headers }) + } + } + const body = await readBody(request) console.log(body) if (body.status === undefined || body.visibility === undefined) { @@ -88,7 +104,6 @@ export async function handleRequest( extraProperties.inReplyTo = inReplyToObject[originalObjectIdSymbol] || inReplyToObject.id.toString() } - const domain = new URL(request.url).hostname const content = enrichStatus(body.status) const mentions = await getMentions(body.status, domain) if (mentions.length > 0) { @@ -116,12 +131,12 @@ export async function handleRequest( } } + if (idempotencyKey !== null) { + await idempotency.insertKey(db, idempotencyKey, note) + } + await timeline.pregenerateTimelines(domain, db, cache, connectedActor) const res = await toMastodonStatusFromObject(db, note, domain) - const headers = { - ...cors(), - 'content-type': 'application/json; charset=utf-8', - } return new Response(JSON.stringify(res), { headers }) } diff --git a/functions/api/v1/statuses/[id].ts b/functions/api/v1/statuses/[id].ts index 0819d01..870be17 100644 --- a/functions/api/v1/statuses/[id].ts +++ b/functions/api/v1/statuses/[id].ts @@ -49,6 +49,7 @@ async function deleteNote(db: D1Database, note: Note) { db.prepare('DELETE FROM actor_favourites WHERE object_id=?').bind(nodeId), db.prepare('DELETE FROM actor_reblogs WHERE object_id=?').bind(nodeId), db.prepare('DELETE FROM actor_replies WHERE object_id=?1 OR in_reply_to_object_id=?1').bind(nodeId), + db.prepare('DELETE FROM idempotency_keys WHERE object_id=?').bind(nodeId), db.prepare('DELETE FROM objects WHERE id=?').bind(nodeId), ] diff --git a/migrations/0005_add_idempotency_keys.sql b/migrations/0005_add_idempotency_keys.sql new file mode 100644 index 0000000..5e5c72c --- /dev/null +++ b/migrations/0005_add_idempotency_keys.sql @@ -0,0 +1,9 @@ +-- Migration number: 0005 2023-02-07T10:57:21.848Z + +CREATE TABLE IF NOT EXISTS idempotency_keys ( + key TEXT PRIMARY KEY, + object_id TEXT NOT NULL, + expires_at DATETIME NOT NULL, + + FOREIGN KEY(object_id) REFERENCES objects(id) +);