From 4b04cc92f1df1a1888dfde893585c1a729dbbc65 Mon Sep 17 00:00:00 2001 From: Nolan Lawson Date: Sun, 21 Jan 2018 17:18:56 -0800 Subject: [PATCH] refactor database --- routes/_components/Timeline.html | 4 +- .../{cleanup.js => cleanupTimelines.js} | 27 +++++--- routes/_utils/database/database.js | 38 ++--------- .../_utils/database/databaseInsideWorker.js | 65 +++++++++++++++++++ routes/_utils/database/knownDbs.js | 21 ++++++ routes/_utils/database/meta.js | 29 +++++++++ .../database/{shared.js => timelines.js} | 20 +++--- routes/_utils/mastodon/user.js | 2 +- routes/settings/instances/[instanceName].html | 26 +++++--- routes/settings/instances/add.html | 7 +- templates/service-worker.js | 1 - 11 files changed, 171 insertions(+), 69 deletions(-) rename routes/_utils/database/{cleanup.js => cleanupTimelines.js} (63%) create mode 100644 routes/_utils/database/databaseInsideWorker.js create mode 100644 routes/_utils/database/knownDbs.js create mode 100644 routes/_utils/database/meta.js rename routes/_utils/database/{shared.js => timelines.js} (54%) diff --git a/routes/_components/Timeline.html b/routes/_components/Timeline.html index c284a0af..67743028 100644 --- a/routes/_components/Timeline.html +++ b/routes/_components/Timeline.html @@ -20,13 +20,11 @@ import LoadingFooter from './LoadingFooter.html' import VirtualList from './VirtualList.html' import { splice, push } from 'svelte-extras' - import worker from 'workerize-loader!../_utils/database/database' import { mergeStatuses } from '../_utils/statuses' import { mark, stop } from '../_utils/marks' import { timelines } from '../_static/timelines' import { toast } from '../_utils/toast' - - const database = worker() + import { database } from '../_utils/database/database' const FETCH_LIMIT = 20 diff --git a/routes/_utils/database/cleanup.js b/routes/_utils/database/cleanupTimelines.js similarity index 63% rename from routes/_utils/database/cleanup.js rename to routes/_utils/database/cleanupTimelines.js index 499abc02..746c421b 100644 --- a/routes/_utils/database/cleanup.js +++ b/routes/_utils/database/cleanupTimelines.js @@ -1,15 +1,15 @@ -import keyval from "idb-keyval" +import { getKnownDbs } from './knownDbs' import debounce from 'lodash/debounce' -import { OBJECT_STORE, getDatabase } from './shared' +import { TIMELINE_STORE, getTimelineDatabase } from './timelines' const MAX_NUM_STORED_STATUSES = 1000 const CLEANUP_INTERVAL = 60000 async function cleanup(instanceName, timeline) { - const db = await getDatabase(instanceName, timeline) + const db = await getTimelineDatabase(instanceName, timeline) return await new Promise((resolve, reject) => { - const tx = db.transaction(OBJECT_STORE, 'readwrite') - const store = tx.objectStore(OBJECT_STORE) + const tx = db.transaction(TIMELINE_STORE, 'readwrite') + const store = tx.objectStore(TIMELINE_STORE) const index = store.index('pinafore_id_as_negative_big_int') store.count().onsuccess = (e) => { @@ -37,11 +37,18 @@ export const cleanupOldStatuses = debounce(async () => { if (process.env.NODE_ENV !== 'production') { console.log('cleanupOldStatuses') } - let knownDbs = (await keyval.get('known_dbs')) || {} - let dbNames = Object.keys(knownDbs) - for (let dbName of dbNames) { - let [ instanceName, timeline ] = knownDbs[dbName] - await cleanup(instanceName, timeline) + + let knownDbs = await getKnownDbs() + let instanceNames = Object.keys(knownDbs) + for (let instanceName of instanceNames) { + let knownDbsForInstance = knownDbs[instanceName] || [] + for (let knownDb of knownDbsForInstance) { + let {type, dbName} = knownDb + if (type !== 'timeline') { + continue + } + await cleanup(instanceName, dbName) + } } if (process.env.NODE_ENV !== 'production') { console.log('done cleanupOldStatuses') diff --git a/routes/_utils/database/database.js b/routes/_utils/database/database.js index c43d802a..ca06ed3c 100644 --- a/routes/_utils/database/database.js +++ b/routes/_utils/database/database.js @@ -1,36 +1,10 @@ -import { cleanupOldStatuses } from './cleanup' -import { OBJECT_STORE, getDatabase } from './shared' -import { toReversePaddedBigInt, transformStatusForStorage } from './utils' +import worker from 'workerize-loader!./databaseInsideWorker' -export async function getTimeline(instanceName, timeline, maxId = null, limit = 20) { - const db = await getDatabase(instanceName, timeline) - return await new Promise((resolve, reject) => { - const tx = db.transaction(OBJECT_STORE, 'readonly') - const store = tx.objectStore(OBJECT_STORE) - const index = store.index('pinafore_id_as_negative_big_int') - let sinceAsNegativeBigInt = maxId ? toReversePaddedBigInt(maxId) : null - let query = sinceAsNegativeBigInt ? IDBKeyRange.lowerBound(sinceAsNegativeBigInt, false) : null +import * as databaseInsideWorker from './databaseInsideWorker' - let res - index.getAll(query, limit).onsuccess = (e) => { - res = e.target.result - } +// workerize-loader causes weirdness during development +let database = process.browser && process.env.NODE_ENV === 'production' ? worker() : databaseInsideWorker - tx.oncomplete = () => resolve(res) - tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message) - }) -} - -export async function insertStatuses(instanceName, timeline, statuses) { - cleanupOldStatuses() - const db = await getDatabase(instanceName, timeline) - return await new Promise((resolve, reject) => { - const tx = db.transaction(OBJECT_STORE, 'readwrite') - const store = tx.objectStore(OBJECT_STORE) - for (let status of statuses) { - store.put(transformStatusForStorage(status)) - } - tx.oncomplete = () => resolve() - tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message) - }) +export { + database } \ No newline at end of file diff --git a/routes/_utils/database/databaseInsideWorker.js b/routes/_utils/database/databaseInsideWorker.js new file mode 100644 index 00000000..54243c5b --- /dev/null +++ b/routes/_utils/database/databaseInsideWorker.js @@ -0,0 +1,65 @@ +import { META_STORE, getMetaDatabase } from './meta' +import { cleanupOldStatuses } from './cleanupTimelines' +import { TIMELINE_STORE, getTimelineDatabase } from './timelines' +import { toReversePaddedBigInt, transformStatusForStorage } from './utils' + +export async function getTimeline(instanceName, timeline, maxId = null, limit = 20) { + const db = await getTimelineDatabase(instanceName, timeline) + return await new Promise((resolve, reject) => { + const tx = db.transaction(TIMELINE_STORE, 'readonly') + const store = tx.objectStore(TIMELINE_STORE) + const index = store.index('pinafore_id_as_negative_big_int') + let sinceAsNegativeBigInt = maxId ? toReversePaddedBigInt(maxId) : null + let query = sinceAsNegativeBigInt ? IDBKeyRange.lowerBound(sinceAsNegativeBigInt, false) : null + + let res + index.getAll(query, limit).onsuccess = (e) => { + res = e.target.result + } + + tx.oncomplete = () => resolve(res) + tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message) + }) +} + +export async function insertStatuses(instanceName, timeline, statuses) { + const db = await getTimelineDatabase(instanceName, timeline) + await new Promise((resolve, reject) => { + const tx = db.transaction(TIMELINE_STORE, 'readwrite') + const store = tx.objectStore(TIMELINE_STORE) + for (let status of statuses) { + store.put(transformStatusForStorage(status)) + } + tx.oncomplete = () => resolve() + tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message) + }) + /* no await */ cleanupOldStatuses() +} + +export async function setInstanceVerifyCredentials(instanceName, verifyCredentials) { + const db = await getMetaDatabase(instanceName) + return await new Promise((resolve, reject) => { + const tx = db.transaction(META_STORE, 'readwrite') + const store = tx.objectStore(META_STORE) + store.put({ + key: 'verifyCredentials', + value: verifyCredentials + }) + tx.oncomplete = () => resolve() + tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message) + }) +} + +export async function getInstanceVerifyCredentials(instanceName, verifyCredentials) { + const db = await getMetaDatabase(instanceName) + return await new Promise((resolve, reject) => { + const tx = db.transaction(META_STORE, 'readwrite') + const store = tx.objectStore(META_STORE) + let res + store.get('verifyCredentials').onsuccess = (e) => { + res = e.target.result && e.target.result.value + } + tx.oncomplete = () => resolve(res) + tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message) + }) +} \ No newline at end of file diff --git a/routes/_utils/database/knownDbs.js b/routes/_utils/database/knownDbs.js new file mode 100644 index 00000000..b5d5f911 --- /dev/null +++ b/routes/_utils/database/knownDbs.js @@ -0,0 +1,21 @@ +import keyval from "idb-keyval" + +export async function addKnownDb(instanceName, type, dbName) { + let knownDbs = (await keyval.get('known_dbs')) || {} + if (!knownDbs[instanceName]) { + knownDbs[instanceName] = [] + } + if (!knownDbs[instanceName].some(db => db.type === type && db.dbName === dbName)) { + knownDbs[instanceName].push({type, dbName}) + } + await keyval.set('known_dbs', knownDbs) +} + +export async function getKnownDbs() { + return (await keyval.get('known_dbs')) || {} +} + +export async function getKnownDbsForInstance(instanceName) { + let knownDbs = await getKnownDbs() + return knownDbs[instanceName] || [] +} \ No newline at end of file diff --git a/routes/_utils/database/meta.js b/routes/_utils/database/meta.js new file mode 100644 index 00000000..2b087d20 --- /dev/null +++ b/routes/_utils/database/meta.js @@ -0,0 +1,29 @@ +import { addKnownDb } from './knownDbs' + +const databaseCache = {} +export const META_STORE = 'meta' + +export function getMetaDatabase(instanceName) { + const key = `${instanceName}_${META_STORE}` + if (databaseCache[key]) { + return Promise.resolve(databaseCache[key]) + } + + let dbName = key + + addKnownDb(instanceName, 'meta', dbName) + + databaseCache[key] = new Promise((resolve, reject) => { + let req = indexedDB.open(dbName, 1) + req.onerror = reject + req.onblocked = () => { + console.log('idb blocked') + } + req.onupgradeneeded = () => { + let db = req.result; + db.createObjectStore(META_STORE, {keyPath: 'key'}) + } + req.onsuccess = () => resolve(req.result) + }) + return databaseCache[key] +} \ No newline at end of file diff --git a/routes/_utils/database/shared.js b/routes/_utils/database/timelines.js similarity index 54% rename from routes/_utils/database/shared.js rename to routes/_utils/database/timelines.js index 1e088833..64b746e2 100644 --- a/routes/_utils/database/shared.js +++ b/routes/_utils/database/timelines.js @@ -1,25 +1,21 @@ -import keyval from "idb-keyval" +import { addKnownDb } from './knownDbs' const databaseCache = {} -export const OBJECT_STORE = 'statuses' +export const TIMELINE_STORE = 'statuses' -export function createDbName(instanceName, timeline) { - return `${OBJECT_STORE}_${instanceName}_${timeline}` +export function createTimelineDbName(instanceName, timeline) { + return `${instanceName}_timeline_${timeline}` } -export function getDatabase(instanceName, timeline) { +export function getTimelineDatabase(instanceName, timeline) { const key = `${instanceName}_${timeline}` if (databaseCache[key]) { return Promise.resolve(databaseCache[key]) } - let dbName = createDbName(instanceName, timeline) + let dbName = createTimelineDbName(instanceName, timeline) - keyval.get('known_dbs').then(knownDbs => { - knownDbs = knownDbs || {} - knownDbs[dbName] = [instanceName, timeline] - keyval.set('known_dbs', knownDbs) - }) + addKnownDb(instanceName, 'timeline', dbName) databaseCache[key] = new Promise((resolve, reject) => { let req = indexedDB.open(dbName, 1) @@ -29,7 +25,7 @@ export function getDatabase(instanceName, timeline) { } req.onupgradeneeded = () => { let db = req.result; - let oStore = db.createObjectStore(OBJECT_STORE, { + let oStore = db.createObjectStore(TIMELINE_STORE, { keyPath: 'id' }) oStore.createIndex('pinafore_id_as_negative_big_int', 'pinafore_id_as_negative_big_int') diff --git a/routes/_utils/mastodon/user.js b/routes/_utils/mastodon/user.js index 09c1b00a..056f8620 100644 --- a/routes/_utils/mastodon/user.js +++ b/routes/_utils/mastodon/user.js @@ -1,7 +1,7 @@ import { get } from '../ajax' import { basename } from './utils' -export function getThisUserAccount(instanceName, accessToken) { +export function getVerifyCredentials(instanceName, accessToken) { let url = `${basename(instanceName)}/api/v1/accounts/verify_credentials` return get(url, { 'Authorization': `Bearer ${accessToken}` diff --git a/routes/settings/instances/[instanceName].html b/routes/settings/instances/[instanceName].html index d4b53c42..3f30c844 100644 --- a/routes/settings/instances/[instanceName].html +++ b/routes/settings/instances/[instanceName].html @@ -6,14 +6,14 @@

{{params.instanceName}}

- {{#if instanceUserAccount}} + {{#if verifyCredentials}}

Logged in as:

- Profile picture for {{'@' + instanceUserAccount.acct}} + Profile picture for {{'@' + verifyCredentials.acct}} {{'@' + instanceUserAccount.acct}} - {{instanceUserAccount.display_name}} + href="{{verifyCredentials.url}}">{{'@' + verifyCredentials.acct}} + {{verifyCredentials.display_name}}

Theme:

@@ -97,7 +97,8 @@ import { store } from '../../_utils/store' import Layout from '../../_components/Layout.html' import SettingsLayout from '../_components/SettingsLayout.html' - import { getThisUserAccount } from '../../_utils/mastodon/user' + import { getVerifyCredentials } from '../../_utils/mastodon/user' + import { database } from '../../_utils/database/database' import { themes } from '../../_static/themes' import { switchToTheme } from '../../_utils/themeEngine' import { goto } from 'sapper/runtime.js' @@ -117,9 +118,18 @@ let loggedInInstances = this.store.get('loggedInInstances') let instanceThemes = this.store.get('instanceThemes') let instanceData = loggedInInstances[instanceName] - let instanceUserAccount = await getThisUserAccount(instanceName, instanceData.access_token) + let verifyCredentials = await database.getInstanceVerifyCredentials(instanceName) + if (verifyCredentials) { // update + getVerifyCredentials(instanceName, instanceData.access_token).then(verifyCredentials => { + database.setInstanceVerifyCredentials(instanceName, verifyCredentials) + }) + } else { + verifyCredentials = await getVerifyCredentials(instanceName, instanceData.access_token) + database.setInstanceVerifyCredentials(instanceName, verifyCredentials) + verifyCredentials = verifyCredentials + } this.set({ - instanceUserAccount: instanceUserAccount, + verifyCredentials: verifyCredentials, selectedTheme: instanceThemes[instanceName] }) }, diff --git a/routes/settings/instances/add.html b/routes/settings/instances/add.html index 39352dd8..67acc7d4 100644 --- a/routes/settings/instances/add.html +++ b/routes/settings/instances/add.html @@ -72,11 +72,12 @@ import Layout from '../../_components/Layout.html'; import SettingsLayout from '../_components/SettingsLayout.html' import { registerApplication, generateAuthLink, getAccessTokenFromAuthCode } from '../../_utils/mastodon/oauth' - import { getThisUserAccount } from '../../_utils/mastodon/user' + import { getVerifyCredentials } from '../../_utils/mastodon/user' import { store } from '../../_utils/store' import { goto } from 'sapper/runtime.js' import { switchToTheme } from '../../_utils/themeEngine' import LoadingMask from '../../_components/LoadingMask' + import { database } from '../../_utils/database/database' const REDIRECT_URI = (typeof location !== 'undefined' ? location.origin : 'https://pinafore.social') + '/settings/instances/add' @@ -177,7 +178,9 @@ this.store.save() switchToTheme('default') // fire off request for account so it's cached in the SW - getThisUserAccount(currentRegisteredInstanceName, instanceData.access_token) + getVerifyCredentials(currentRegisteredInstanceName, instanceData.access_token).then(verifyCredentials => { + database.setInstanceVerifyCredentials(currentRegisteredInstanceName, verifyCredentials) + }) goto('/') } } diff --git a/templates/service-worker.js b/templates/service-worker.js index 3a09102f..c587ff2a 100644 --- a/templates/service-worker.js +++ b/templates/service-worker.js @@ -42,7 +42,6 @@ const NETWORK_ONLY = [ ] const CACHE_FIRST = [ - '/api/v1/accounts/verify_credentials', '/system/accounts/avatars' ]