MOW-133: add idempotency for posting

pull/212/head
Sven Sauleau 2023-02-07 11:49:39 +00:00
rodzic 9d4415fbb0
commit e13c0fb670
6 zmienionych plików z 122 dodań i 9 usunięć

Wyświetl plik

@ -0,0 +1,51 @@
import type { APObject } from 'wildebeest/backend/src/activitypub/objects'
import {
mastodonIdSymbol,
originalActorIdSymbol,
originalObjectIdSymbol,
} from 'wildebeest/backend/src/activitypub/objects'
export async function insertKey(db: D1Database, key: string, obj: APObject): Promise<void> {
const query = `
INSERT INTO idempotency_keys (key, object_id, expires_at)
VALUES (?1, ?2, datetime('now', '+1 hour'))
`
const { success, error } = await db.prepare(query).bind(key, obj.id.toString()).run()
if (!success) {
throw new Error('SQL error: ' + error)
}
}
export async function hasKey(db: D1Database, key: string): Promise<APObject | null> {
const query = `
SELECT objects.*
FROM idempotency_keys
INNER JOIN objects ON objects.id = idempotency_keys.object_id
WHERE idempotency_keys.key = ?1 AND expires_at >= datetime()
`
const { results, success, error } = await db.prepare(query).bind(key).all<any>()
if (!success) {
throw new Error('SQL error: ' + error)
}
if (!results || results.length === 0) {
return null
}
const result = results[0]
const properties = JSON.parse(result.properties)
return {
published: new Date(result.cdate).toISOString(),
...properties,
type: result.type,
id: new URL(result.id),
[mastodonIdSymbol]: result.mastodon_id,
[originalActorIdSymbol]: result.original_actor_id,
[originalObjectIdSymbol]: result.original_object_id,
} as APObject
}

Wyświetl plik

@ -27,7 +27,6 @@ describe('ActivityPub', () => {
assert.equal(request.method, 'POST')
const data = await request.json()
receivedActivity = data
console.log({ receivedActivity })
return new Response('')
}

Wyświetl plik

@ -115,7 +115,6 @@ describe('Mastodon APIs', () => {
const data = await res.json<any>()
const cachedData = await cache.get<any>(actor.id + '/timeline/home')
console.log({ cachedData })
assert(cachedData)
assert.equal(cachedData.length, 1)
assert.equal(cachedData[0].id, data.id)
@ -875,5 +874,44 @@ describe('Mastodon APIs', () => {
assert.equal(queue.messages[1].actorId, actor.id.toString())
assert.equal(queue.messages[1].toActorId, actor3.id.toString())
})
test('create duplicate statuses idempotency', async () => {
const db = await makeDB()
const queue = makeQueue()
const actor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com')
const idempotencyKey = 'abcd'
const body = { status: 'my status', visibility: 'public' }
const req = () =>
new Request('https://example.com', {
method: 'POST',
headers: {
'content-type': 'application/json',
'idempotency-key': idempotencyKey,
},
body: JSON.stringify(body),
})
const res1 = await statuses.handleRequest(req(), db, actor, userKEK, queue, cache)
assert.equal(res1.status, 200)
const data1 = await res1.json()
const res2 = await statuses.handleRequest(req(), db, actor, userKEK, queue, cache)
assert.equal(res2.status, 200)
const data2 = await res2.json()
assert.deepEqual(data1, data2)
{
const row = await db.prepare(`SELECT count(*) as count FROM objects`).first<{ count: number }>()
assert.equal(row.count, 1)
}
{
const row = await db.prepare(`SELECT count(*) as count FROM idempotency_keys`).first<{ count: number }>()
assert.equal(row.count, 1)
}
})
})
})

Wyświetl plik

@ -1,5 +1,6 @@
// https://docs.joinmastodon.org/methods/statuses/#create
import type { Note } from 'wildebeest/backend/src/activitypub/objects/note'
import { cors } from 'wildebeest/backend/src/utils/cors'
import type { APObject } from 'wildebeest/backend/src/activitypub/objects'
import { insertReply } from 'wildebeest/backend/src/mastodon/reply'
@ -21,6 +22,7 @@ import { toMastodonStatusFromObject } from 'wildebeest/backend/src/mastodon/stat
import type { Cache } from 'wildebeest/backend/src/cache'
import { cacheFromEnv } from 'wildebeest/backend/src/cache'
import { enrichStatus } from 'wildebeest/backend/src/mastodon/microformats'
import * as idempotency from 'wildebeest/backend/src/mastodon/idempotency'
import { newMention } from 'wildebeest/backend/src/activitypub/objects/mention'
import { originalObjectIdSymbol } from 'wildebeest/backend/src/activitypub/objects'
@ -45,12 +47,26 @@ export async function handleRequest(
queue: Queue<DeliverMessageBody>,
cache: Cache
): Promise<Response> {
// TODO: implement Idempotency-Key
if (request.method !== 'POST') {
return new Response('', { status: 400 })
}
const domain = new URL(request.url).hostname
const headers = {
...cors(),
'content-type': 'application/json; charset=utf-8',
}
const idempotencyKey = request.headers.get('Idempotency-Key')
if (idempotencyKey !== null) {
const maybeObject = await idempotency.hasKey(db, idempotencyKey)
if (maybeObject !== null) {
const res = await toMastodonStatusFromObject(db, maybeObject as Note, domain)
return new Response(JSON.stringify(res), { headers })
}
}
const body = await readBody<StatusCreate>(request)
console.log(body)
if (body.status === undefined || body.visibility === undefined) {
@ -88,7 +104,6 @@ export async function handleRequest(
extraProperties.inReplyTo = inReplyToObject[originalObjectIdSymbol] || inReplyToObject.id.toString()
}
const domain = new URL(request.url).hostname
const content = enrichStatus(body.status)
const mentions = await getMentions(body.status, domain)
if (mentions.length > 0) {
@ -116,12 +131,12 @@ export async function handleRequest(
}
}
if (idempotencyKey !== null) {
await idempotency.insertKey(db, idempotencyKey, note)
}
await timeline.pregenerateTimelines(domain, db, cache, connectedActor)
const res = await toMastodonStatusFromObject(db, note, domain)
const headers = {
...cors(),
'content-type': 'application/json; charset=utf-8',
}
return new Response(JSON.stringify(res), { headers })
}

Wyświetl plik

@ -49,6 +49,7 @@ async function deleteNote(db: D1Database, note: Note) {
db.prepare('DELETE FROM actor_favourites WHERE object_id=?').bind(nodeId),
db.prepare('DELETE FROM actor_reblogs WHERE object_id=?').bind(nodeId),
db.prepare('DELETE FROM actor_replies WHERE object_id=?1 OR in_reply_to_object_id=?1').bind(nodeId),
db.prepare('DELETE FROM idempotency_keys WHERE object_id=?').bind(nodeId),
db.prepare('DELETE FROM objects WHERE id=?').bind(nodeId),
]

Wyświetl plik

@ -0,0 +1,9 @@
-- Migration number: 0005 2023-02-07T10:57:21.848Z
CREATE TABLE IF NOT EXISTS idempotency_keys (
key TEXT PRIMARY KEY,
object_id TEXT NOT NULL,
expires_at DATETIME NOT NULL,
FOREIGN KEY(object_id) REFERENCES objects(id)
);