From c0918ccdc36d569dcc574f4e2c29eb374f6aa21d Mon Sep 17 00:00:00 2001 From: Nolan Lawson Date: Tue, 5 Mar 2019 20:25:10 -0800 Subject: [PATCH] fix: fix pleroma snowflake IDs for real (#1084) fixes #1082 --- src/routes/_database/constants.js | 4 +- src/routes/_utils/statusIdSorting.js | 18 ++- tests/unit/test-database.js | 2 +- tests/unit/test-id-sorting.js | 163 ++++++++++++++++++++++++++- 4 files changed, 173 insertions(+), 14 deletions(-) diff --git a/src/routes/_database/constants.js b/src/routes/_database/constants.js index 56883e9f..6d72f393 100644 --- a/src/routes/_database/constants.js +++ b/src/routes/_database/constants.js @@ -16,8 +16,8 @@ export const USERNAME_LOWERCASE = '__pinafore_acct_lc' export const DB_VERSION_INITIAL = 9 export const DB_VERSION_SEARCH_ACCOUNTS = 10 -export const DB_VERSION_SNOWFLAKE_IDS = 11 +export const DB_VERSION_SNOWFLAKE_IDS = 12 // 11 skipped because of mistake deployed to dev.pinafore.social // Using an object for these so that unit tests can change them -export const DB_VERSION_CURRENT = { version: 11 } +export const DB_VERSION_CURRENT = { version: 12 } export const CURRENT_TIME = { now: () => Date.now() } diff --git a/src/routes/_utils/statusIdSorting.js b/src/routes/_utils/statusIdSorting.js index 960df661..f7b585f9 100644 --- a/src/routes/_utils/statusIdSorting.js +++ b/src/routes/_utils/statusIdSorting.js @@ -4,13 +4,12 @@ import { padStart } from './lodash-lite' -// Unfortunately base62 ordering is not the same as JavaScript's default ASCII ordering, -// used both for JS string comparisons as well as IndexedDB ordering. -const BASE62_ALPHABET = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' +// Pleroma uses the 0-9A-Za-z alphabet for base62, which is the same as ASCII, which +// is the same as JavaScript sort order and IndexedDB order. +const MIN_CHAR_CODE = 48 // '0'.charCodeAt(0) +const MAX_CHAR_CODE = 122 // 'z'.charCodeAt(0) const MAX_ID_LENGTH = 30 // assume that Mastodon/Pleroma IDs won't get any bigger than this -const BASE62_LOOKUP = new Map(BASE62_ALPHABET.split('').map((char, i) => ([char, i]))) - export function zeroPad (str, toSize) { return padStart(str, toSize, '0') } @@ -20,13 +19,12 @@ export function toPaddedBigInt (id) { } export function toReversePaddedBigInt (id) { - let padded = zeroPad(id, MAX_ID_LENGTH) + let padded = toPaddedBigInt(id) let reversed = '' for (let i = 0; i < padded.length; i++) { - let char = padded.charAt(i) - let idx = BASE62_LOOKUP.get(char) - let reverseIdx = BASE62_ALPHABET.length - 1 - idx - reversed += BASE62_ALPHABET[reverseIdx] + let charCode = padded.charCodeAt(i) + let inverseCharCode = MIN_CHAR_CODE + MAX_CHAR_CODE - charCode + reversed += String.fromCharCode(inverseCharCode) } return reversed } diff --git a/tests/unit/test-database.js b/tests/unit/test-database.js index f2176395..c985206c 100644 --- a/tests/unit/test-database.js +++ b/tests/unit/test-database.js @@ -153,7 +153,7 @@ describe('test-database.js', function () { await deleteDatabase(INSTANCE_NAME) }) - it('migrates from v10 to v11', async () => { + it('migrates to snowflake IDs', async () => { // open the db using the old version DB_VERSION_CURRENT.version = DB_VERSION_SEARCH_ACCOUNTS await getDatabase(INSTANCE_NAME) diff --git a/tests/unit/test-id-sorting.js b/tests/unit/test-id-sorting.js index 3ef519f7..00b1de13 100644 --- a/tests/unit/test-id-sorting.js +++ b/tests/unit/test-id-sorting.js @@ -13,6 +13,15 @@ function gt (a, b) { } describe('test-id-sorting.js', () => { + it('basic id sorting', () => { + assert.deepStrictEqual(toPaddedBigInt('0'), '000000000000000000000000000000') + assert.deepStrictEqual(toPaddedBigInt('1'), '000000000000000000000000000001') + assert.deepStrictEqual(toPaddedBigInt('z'), '00000000000000000000000000000z') + assert.deepStrictEqual(toReversePaddedBigInt('0'), 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz') + assert.deepStrictEqual(toReversePaddedBigInt('1'), 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzy') + assert.deepStrictEqual(toReversePaddedBigInt('z'), 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzz0') + }) + it('can sort mastodon IDs correctly', () => { let id1 = '1' let id2 = '2' @@ -94,7 +103,75 @@ describe('test-id-sorting.js', () => { assert.deepStrictEqual(toReversePaddedBigInt(id5), toReversePaddedBigInt(id5)) }) - it('can sort pleroma ids - more examples', () => { + it('can sort base62 IDs correctly 2', () => { + let id1 = '0' + let id2 = 'A' + let id3 = 'T' + let id4 = 'a' + let id5 = 'z' + + lt(toPaddedBigInt(id1), toPaddedBigInt(id2)) + lt(toPaddedBigInt(id2), toPaddedBigInt(id3)) + lt(toPaddedBigInt(id3), toPaddedBigInt(id4)) + lt(toPaddedBigInt(id4), toPaddedBigInt(id5)) + + lt(toPaddedBigInt(id1), toPaddedBigInt(id5)) + lt(toPaddedBigInt(id2), toPaddedBigInt(id5)) + lt(toPaddedBigInt(id3), toPaddedBigInt(id5)) + lt(toPaddedBigInt(id2), toPaddedBigInt(id4)) + + assert.deepStrictEqual(toPaddedBigInt(id1), toPaddedBigInt(id1)) + assert.deepStrictEqual(toPaddedBigInt(id2), toPaddedBigInt(id2)) + assert.deepStrictEqual(toPaddedBigInt(id3), toPaddedBigInt(id3)) + assert.deepStrictEqual(toPaddedBigInt(id4), toPaddedBigInt(id4)) + assert.deepStrictEqual(toPaddedBigInt(id5), toPaddedBigInt(id5)) + + gt(toReversePaddedBigInt(id1), toReversePaddedBigInt(id2)) + gt(toReversePaddedBigInt(id2), toReversePaddedBigInt(id3)) + gt(toReversePaddedBigInt(id3), toReversePaddedBigInt(id4)) + gt(toReversePaddedBigInt(id4), toReversePaddedBigInt(id5)) + + gt(toReversePaddedBigInt(id1), toReversePaddedBigInt(id5)) + gt(toReversePaddedBigInt(id2), toReversePaddedBigInt(id5)) + gt(toReversePaddedBigInt(id3), toReversePaddedBigInt(id5)) + gt(toReversePaddedBigInt(id2), toReversePaddedBigInt(id4)) + + assert.deepStrictEqual(toReversePaddedBigInt(id1), toReversePaddedBigInt(id1)) + assert.deepStrictEqual(toReversePaddedBigInt(id2), toReversePaddedBigInt(id2)) + assert.deepStrictEqual(toReversePaddedBigInt(id3), toReversePaddedBigInt(id3)) + assert.deepStrictEqual(toReversePaddedBigInt(id4), toReversePaddedBigInt(id4)) + assert.deepStrictEqual(toReversePaddedBigInt(id5), toReversePaddedBigInt(id5)) + }) + + it('can sort base62 IDs correctly 3', () => { + let id1 = 'a' + let id2 = 'z' + let id3 = 'a0' + let id4 = 'xx0' + let id5 = 'a000' + + lt(toPaddedBigInt(id1), toPaddedBigInt(id2)) + lt(toPaddedBigInt(id2), toPaddedBigInt(id3)) + lt(toPaddedBigInt(id3), toPaddedBigInt(id4)) + lt(toPaddedBigInt(id4), toPaddedBigInt(id5)) + + lt(toPaddedBigInt(id1), toPaddedBigInt(id5)) + lt(toPaddedBigInt(id2), toPaddedBigInt(id5)) + lt(toPaddedBigInt(id3), toPaddedBigInt(id5)) + lt(toPaddedBigInt(id2), toPaddedBigInt(id4)) + + gt(toReversePaddedBigInt(id1), toReversePaddedBigInt(id2)) + gt(toReversePaddedBigInt(id2), toReversePaddedBigInt(id3)) + gt(toReversePaddedBigInt(id3), toReversePaddedBigInt(id4)) + gt(toReversePaddedBigInt(id4), toReversePaddedBigInt(id5)) + + gt(toReversePaddedBigInt(id1), toReversePaddedBigInt(id5)) + gt(toReversePaddedBigInt(id2), toReversePaddedBigInt(id5)) + gt(toReversePaddedBigInt(id3), toReversePaddedBigInt(id5)) + gt(toReversePaddedBigInt(id2), toReversePaddedBigInt(id4)) + }) + + it('can sort pleroma ids', () => { // these are already in base62 sorted order let ids = [ '9gP7cpqqJWyp93GxRw', @@ -126,4 +203,88 @@ describe('test-id-sorting.js', () => { gt(toReversePaddedBigInt(prev), toReversePaddedBigInt(next)) } }) + + it('can sort pleroma ids 2', () => { + let ids = [ + '9gTv5mTEiXL6ZpqYHg', + '9gTv5mK1Gny07FXBuy', + '9gTv5kXlshmKbJx94i', + '9gTv5f9TvaNVsXYB1M', + '9gTv5Fj2SlN71HzCpk', + '9gTv5DibvNqCnNlptA', + '9gTv4ttvbb0hKguaki', + '9gTv4n17CTbFyxDjU0', + '9gTv43wGndzCAtbjBQ', + '9gTv3zP9ep7W725E0m', + '9gTv3mQRuhQnrHaZiy', + '9gTv3mOK2bjJkgasPQ', + '9gTv3kpXqQJiuXJaYy', + '9gTv3JliSYDAyqJeLY', + '9gTv36jSvbgeY6AAXA', + '9gTv2udLuVfP1fD7L6', + '9gTv2gbQ4tnnVcCNNo', + '9gTv2FSH0nRXJQiBsW', + '9gTv1tfzz5LllcxqDY', + '9gTv1t1EQejxjBtHfs' + ].reverse() + + for (let i = 1; i < ids.length; i++) { + let prev = ids[i - 1] + let next = ids[i] + lt(toPaddedBigInt(prev), toPaddedBigInt(next)) + gt(toReversePaddedBigInt(prev), toReversePaddedBigInt(next)) + } + }) + + it('can sort pleroma ids - 3', () => { + let ids = [ + '9gTCUO2xe7vfkLbHRA', + '9gT35b559J1tLPhGj2', + '9gRax4YxAwDuIdr83U', + '9gQqktJiZ6ha3Tz0fA', + '9gOyRmT0DkfWcmnw7k', + '9gOvJd0nBsvd7a1wUy', + '9gOsyLgqCKARPQQKPo', + '9gOsiEkAFyEnERINOq', + '9gOsHBDvEh0EUoJRom', + '9gOrx9MURrsivilkjw', + '9gOrpQWsjga3rEwmw4', + '9gOriOTFzUcLRjtjkm', + '9gOraOfrLpz3lBynXE', + '9gOrLvdMWe0Ldgudbk', + '9gOrB5vttTXiOaPxCq', + '9gOpJz8uJEGa2Ac47E', + '9gOhCP31JNzWQXjVEu', + '9gOdCdne43SIMZOp8K', + '9gOcJKlz6VfvQRFZyq', + '9gOcGtLW6bmXmchoGG', + '9gOc78e2N27GKriYdM', + '9gOc4BmYFXBvCHIGzw', + '9gObs3Wx6Rt888DCHQ', + '9gObp0SU4J0tjXfSZk', + '9gOaaY4UiJOngVzN6e', + '9gOaAiTbPDbQQTPnpQ', + '9gOZjQH0yaB29SVrIe', + '9gOZTFEsEbV1IMnUDA', + '9gOZQA9yf55sfs2NYu', + '9gOZNK8p5vaulK8yKO', + '9gOZMvOZ6GKe0uJgtU', + '9gOZIXp4ndv9kMd1SC', + '9gOWmmfFsr2obOlJuy', + '9gOWivHDvAoO9egFmK', + '9gOWZDwXfIAH9TNkbQ', + '9gOWUUUkhMq92R7v6m', + '9gOWChzRml0fgE0MJk', + '9gOVO8G0MitkNRdIjw', + '9gNEd8FWme8oJn15Jg', + '9gNEUoR2ZKkgY4Qzce' + ].reverse() + + for (let i = 1; i < ids.length; i++) { + let prev = ids[i - 1] + let next = ids[i] + lt(toPaddedBigInt(prev), toPaddedBigInt(next)) + gt(toReversePaddedBigInt(prev), toReversePaddedBigInt(next)) + } + }) })