proof of concept: assets in indexedDB, not as base64 in store

apps/dotcom/src/utils/cloneAssetForShare.ts

@@ -1,18 +1,32 @@
-import { TLAsset } from 'tldraw'
+import { Editor, TLAsset } from 'tldraw'
 
 export async function cloneAssetForShare(
 	asset: TLAsset,
+	editor: Editor,
 	uploadFileToAsset: (file: File) => Promise<TLAsset>
 ): Promise<TLAsset> {
 	if (asset.type === 'bookmark') return asset
-	if (asset.props.src) {
-		const dataUrlMatch = asset.props.src.match(/data:(.*?)(;base64)?,/)
-		if (!dataUrlMatch) return asset
 
-		const response = await fetch(asset.props.src)
-		const file = new File([await response.blob()], asset.props.name, {
-			type: dataUrlMatch[1] ?? asset.props.mimeType,
-		})
+	if (asset.props.src) {
+		let file: File | undefined
+		if (asset.props.src.startsWith('asset:')) {
+			const blob = await editor.getAssetBlobFromObjectStore(asset)
+			if (blob) {
+				file = new File([blob], asset.props.name, {
+					type: asset.props.mimeType || '',
+				})
+			} else {
+				return asset
+			}
+		} else {
+			const dataUrlMatch = asset.props.src.match(/data:(.*?)(;base64)?,/)
+			if (!dataUrlMatch) return asset
+
+			const response = await fetch(asset.props.src)
+			file = new File([await response.blob()], asset.props.name, {
+				type: dataUrlMatch[1] ?? asset.props.mimeType,
+			})
+		}
 
 		const uploadedAsset = await uploadFileToAsset(file)
 

apps/dotcom/src/utils/sharing.ts

@@ -241,7 +241,7 @@ async function getRoomData(
 			// processed it
 			if (!asset) continue
 
-			data[asset.id] = await cloneAssetForShare(asset, uploadFileToAsset)
+			data[asset.id] = await cloneAssetForShare(asset, editor, uploadFileToAsset)
 			// remove the asset after processing so we don't clone it multiple times
 			assets.delete(asset.id)
 		}

packages/editor/src/lib/TldrawEditor.tsx

@@ -36,6 +36,7 @@ import { useLocalStore } from './hooks/useLocalStore'
 import { useSafariFocusOutFix } from './hooks/useSafariFocusOutFix'
 import { useZoomCss } from './hooks/useZoomCss'
 import { stopEventPropagation } from './utils/dom'
+import { AssetBlobObjectStore } from './utils/sync/AssetBlobObjectStore'
 import { TLStoreWithStatus } from './utils/sync/StoreWithStatus'
 
 /**
@@ -48,9 +49,11 @@ export type TldrawEditorProps = Expand<
 		(
 			| {
 					store: TLStore | TLStoreWithStatus
+					assetBlobStore: AssetBlobObjectStore
 			  }
 			| {
 					store?: undefined
+					assetBlobStore: AssetBlobObjectStore
 					migrations?: readonly MigrationSequence[]
 					snapshot?: StoreSnapshot<TLRecord>
 					initialData?: SerializedStore<TLRecord>
@@ -175,6 +178,10 @@ export const TldrawEditor = memo(function TldrawEditor({
 		components,
 	}
 
+	const assetBlobStore = new AssetBlobObjectStore(
+		'persistenceKey' in rest ? rest.persistenceKey || '' : ''
+	)
+
 	return (
 		<div
 			ref={setContainer}
@@ -193,14 +200,29 @@ export const TldrawEditor = memo(function TldrawEditor({
 							{store ? (
 								store instanceof Store ? (
 									// Store is ready to go, whether externally synced or not
-									<TldrawEditorWithReadyStore {...withDefaults} store={store} user={user} />
+									<TldrawEditorWithReadyStore
+										{...withDefaults}
+										store={store}
+										assetBlobStore={assetBlobStore}
+										user={user}
+									/>
 								) : (
 									// Store is a synced store, so handle syncing stages internally
-									<TldrawEditorWithLoadingStore {...withDefaults} store={store} user={user} />
+									<TldrawEditorWithLoadingStore
+										{...withDefaults}
+										store={store}
+										assetBlobStore={assetBlobStore}
+										user={user}
+									/>
 								)
 							) : (
 								// We have no store (it's undefined) so create one and possibly sync it
-								<TldrawEditorWithOwnStore {...withDefaults} store={store} user={user} />
+								<TldrawEditorWithOwnStore
+									{...withDefaults}
+									store={store}
+									assetBlobStore={assetBlobStore}
+									user={user}
+								/>
 							)}
 						</EditorComponentsProvider>
 					</ContainerProvider>
@@ -287,6 +309,7 @@ function TldrawEditorWithReadyStore({
 	onMount,
 	children,
 	store,
+	assetBlobStore,
 	tools,
 	shapeUtils,
 	bindingUtils,
@@ -309,6 +332,7 @@ function TldrawEditorWithReadyStore({
 	useLayoutEffect(() => {
 		const editor = new Editor({
 			store,
+			assetBlobStore,
 			shapeUtils,
 			bindingUtils,
 			tools,
@@ -329,6 +353,7 @@ function TldrawEditorWithReadyStore({
 		bindingUtils,
 		tools,
 		store,
+		assetBlobStore,
 		user,
 		initialState,
 		inferDarkMode,

packages/editor/src/lib/editor/Editor.ts

@@ -54,6 +54,7 @@ import {
 	JsonObject,
 	PerformanceTracker,
 	Result,
+	WeakCache,
 	annotateError,
 	assert,
 	assertExists,
@@ -116,6 +117,7 @@ import { debugFlags } from '../utils/debug-flags'
 import { getIncrementedName } from '../utils/getIncrementedName'
 import { getReorderingShapesChanges } from '../utils/reorderShapes'
 import { applyRotationToSnapshotShapes, getRotationSnapshot } from '../utils/rotation'
+import { AssetBlobObjectStore } from '../utils/sync/AssetBlobObjectStore'
 import { uniqueId } from '../utils/uniqueId'
 import { BindingUtil, TLBindingUtilConstructor } from './bindings/BindingUtil'
 import { bindingsIndex } from './derivations/bindingsIndex'
@@ -173,6 +175,10 @@ export interface TLEditorOptions {
 	 * from a server or database.
 	 */
 	store: TLStore
+	/**
+	 * Object store for keeping the app's asset data.
+	 */
+	assetBlobStore: AssetBlobObjectStore
 	/**
 	 * An array of shapes to use in the editor. These will be used to create and manage shapes in the editor.
 	 */
@@ -212,6 +218,7 @@ export interface TLEditorOptions {
 export class Editor extends EventEmitter<TLEventMap> {
 	constructor({
 		store,
+		assetBlobStore,
 		user,
 		shapeUtils,
 		bindingUtils,
@@ -224,6 +231,7 @@ export class Editor extends EventEmitter<TLEventMap> {
 		super()
 
 		this.store = store
+		this.assetBlobStore = assetBlobStore
 		this.history = new HistoryManager<TLRecord>({
 			store,
 			annotateError: (error) => {
@@ -652,6 +660,11 @@ export class Editor extends EventEmitter<TLEventMap> {
 	 */
 	readonly store: TLStore
 
+	/**
+	 * The editor's asset blob store.
+	 */
+	readonly assetBlobStore: AssetBlobObjectStore
+
 	/**
 	 * The root state of the statechart.
 	 *
@@ -3669,6 +3682,39 @@ export class Editor extends EventEmitter<TLEventMap> {
 		return this.batch(() => this.store.put(assets))
 	}
 
+	/** @private */
+	private assetBlobCache = new WeakCache<TLAsset, Promise<Blob | undefined>>()
+
+	/**
+	 * Get an asset from the database.
+	 *
+	 * @example
+	 * ```ts
+	 * editor.getAssetBlobInObjectStore('asset1')
+	 * ```
+	 *
+	 * @public
+	 */
+	async getAssetBlobFromObjectStore(asset: TLAsset): Promise<Blob | undefined> {
+		return await this.assetBlobCache.get(asset, () =>
+			this.assetBlobStore.getAssetBlobFromObjectStore({ asset })
+		)
+	}
+
+	/**
+	 * Put an asset into the database.
+	 *
+	 * @example
+	 * ```ts
+	 * editor.putAssetBlobInObjectStore('asset1', <blob>)
+	 * ```
+	 *
+	 * @public
+	 */
+	async putAssetBlobInObjectStore(asset: TLAsset, assetBlob: Blob) {
+		await this.assetBlobStore.putAssetBlobInObjectStore({ asset, assetBlob })
+	}
+
 	/**
 	 * Update one or more assets.
 	 *

packages/editor/src/lib/utils/sync/AssetBlobObjectStore.ts

@@ -0,0 +1,18 @@
+import { TLAsset } from '@tldraw/tlschema'
+import { getAssetFromIndexedDb, storeAssetInIndexedDb } from './indexedDb'
+
+export class AssetBlobObjectStore {
+	constructor(public persistenceKey: string) {}
+
+	async getAssetBlobFromObjectStore({ asset }: { asset: TLAsset }): Promise<Blob | undefined> {
+		return await getAssetFromIndexedDb({ persistenceKey: this.persistenceKey, assetId: asset.id })
+	}
+
+	async putAssetBlobInObjectStore({ asset, assetBlob }: { asset: TLAsset; assetBlob: Blob }) {
+		await storeAssetInIndexedDb({
+			persistenceKey: this.persistenceKey,
+			assetId: asset.id,
+			assetBlob,
+		})
+	}
+}

packages/editor/src/lib/utils/sync/indexedDb.ts

@@ -12,13 +12,14 @@ const Table = {
 	Records: 'records',
 	Schema: 'schema',
 	SessionState: 'session_state',
+	Assets: 'assets',
 } as const
 
 type StoreName = (typeof Table)[keyof typeof Table]
 
 async function withDb<T>(storeId: string, cb: (db: IDBPDatabase<StoreName>) => Promise<T>) {
 	addDbName(storeId)
-	const db = await openDB<StoreName>(storeId, 3, {
+	const db = await openDB<StoreName>(storeId, 4, {
 		upgrade(database) {
 			if (!database.objectStoreNames.contains(Table.Records)) {
 				database.createObjectStore(Table.Records)
@@ -29,6 +30,9 @@ async function withDb<T>(storeId: string, cb: (db: IDBPDatabase<StoreName>) => P
 			if (!database.objectStoreNames.contains(Table.SessionState)) {
 				database.createObjectStore(Table.SessionState)
 			}
+			if (!database.objectStoreNames.contains(Table.Assets)) {
+				database.createObjectStore(Table.Assets)
+			}
 		},
 	})
 	try {
@@ -146,6 +150,43 @@ export async function storeChangesInIndexedDb({
 	})
 }
 
+/** @internal */
+export async function getAssetFromIndexedDb({
+	persistenceKey,
+	assetId,
+}: {
+	persistenceKey: string
+	assetId: string
+}): Promise<Blob | undefined> {
+	const storeId = STORE_PREFIX + persistenceKey
+
+	return await withDb(storeId, async (db) => {
+		const tx = db.transaction([Table.Assets], 'readwrite')
+		const assetsStore = tx.objectStore(Table.Assets)
+		return await assetsStore.get(assetId)
+	})
+}
+
+/** @internal */
+export async function storeAssetInIndexedDb({
+	persistenceKey,
+	assetId,
+	assetBlob,
+}: {
+	persistenceKey: string
+	assetId: string
+	assetBlob: Blob
+}) {
+	const storeId = STORE_PREFIX + persistenceKey
+
+	await withDb(storeId, async (db) => {
+		const tx = db.transaction([Table.Assets], 'readwrite')
+		const assetsStore = tx.objectStore(Table.Assets)
+		await assetsStore.put(assetBlob, assetId)
+		await tx.done
+	})
+}
+
 /** @internal */
 export async function storeSnapshotInIndexedDb({
 	persistenceKey,

packages/tldraw/src/lib/defaultExternalContentHandlers.ts

@@ -1,7 +1,6 @@
 import {
 	AssetRecordType,
 	Editor,
-	FileHelpers,
 	MediaHelpers,
 	TLAsset,
 	TLAssetId,
@@ -97,7 +96,7 @@ export function registerDefaultExternalContentHandlers(
 			typeName: 'asset',
 			props: {
 				name,
-				src: await FileHelpers.blobToDataUrl(file),
+				src: assetId,
 				w: size.w,
 				h: size.h,
 				mimeType: file.type,
@@ -105,6 +104,8 @@ export function registerDefaultExternalContentHandlers(
 			},
 		})
 
+		await editor.putAssetBlobInObjectStore(asset, file)
+
 		return asset
 	})
 

packages/tldraw/src/lib/shapes/image/ImageShapeUtil.tsx

@@ -49,13 +49,28 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 		const [staticFrameSrc, setStaticFrameSrc] = useState('')
 
 		const asset = shape.props.assetId ? this.editor.getAsset(shape.props.assetId) : undefined
+		const [url, setUrl] = useState(asset?.props.src || '')
 
 		const isSelected = shape.id === this.editor.getOnlySelectedShapeId()
+		const editor = this.editor
 
 		useEffect(() => {
-			if (asset?.props.src && 'mimeType' in asset.props && asset?.props.mimeType === 'image/gif') {
+			async function retrieveAsset() {
+				// Retrieve a local image from the DB.
+				if (asset && url?.startsWith('asset:')) {
+					const blob = await editor.getAssetBlobFromObjectStore(asset)
+					if (blob) {
+						const imgURL = URL.createObjectURL(blob)
+						setUrl(imgURL)
+					}
+				}
+			}
+			retrieveAsset()
+		}, [url, asset, editor])
+
+		useEffect(() => {
+			if (asset && url && 'mimeType' in asset.props && asset?.props.mimeType === 'image/gif') {
 				let cancelled = false
-				const url = asset.props.src
 				if (!url) return
 
 				const image = new Image()
@@ -79,7 +94,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 					cancelled = true
 				}
 			}
-		}, [prefersReducedMotion, asset?.props])
+		}, [prefersReducedMotion, url, asset])
 
 		if (asset?.type === 'bookmark') {
 			throw Error("Bookmark assets can't be rendered as images")
@@ -97,7 +112,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 
 		const containerStyle = getCroppedContainerStyle(shape)
 
-		if (!asset?.props.src) {
+		if (!url) {
 			return (
 				<HTMLContainer
 					id={shape.id}
@@ -121,6 +136,8 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 			)
 		}
 
+		if (url.startsWith('asset:')) return null
+
 		return (
 			<>
 				{showCropPreview && (
@@ -130,7 +147,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 							style={{
 								opacity: 0.1,
 								backgroundImage: `url(${
-									!shape.props.playing || reduceMotion ? staticFrameSrc : asset.props.src
+									!shape.props.playing || reduceMotion ? staticFrameSrc : url
 								})`,
 							}}
 							draggable={false}
@@ -146,12 +163,12 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 							className="tl-image"
 							style={{
 								backgroundImage: `url(${
-									!shape.props.playing || reduceMotion ? staticFrameSrc : asset.props.src
+									!shape.props.playing || reduceMotion ? staticFrameSrc : url
 								})`,
 							}}
 							draggable={false}
 						/>
-						{asset.props.isAnimated && !shape.props.playing && (
+						{asset?.props.isAnimated && !shape.props.playing && (
 							<div className="tl-image__tg">GIF</div>
 						)}
 					</div>
@@ -176,6 +193,14 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 		if (!asset) return null
 
 		let src = asset?.props.src || ''
+
+		if (src?.startsWith('asset:')) {
+			const blob = await this.editor.getAssetBlobFromObjectStore(asset)
+			if (blob) {
+				src = (await FileHelpers.blobToDataUrl(blob)) || ''
+			}
+		}
+
 		if (src.startsWith('http') || src.startsWith('/') || src.startsWith('./')) {
 			// If it's a remote image, we need to fetch it and convert it to a data URI
 			src = (await getDataURIFromURL(src)) || ''

packages/tldraw/src/lib/shapes/video/VideoShapeUtil.tsx

@@ -40,9 +40,24 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
 		const { time, playing } = shape.props
 		const isEditing = useIsEditing(shape.id)
 		const prefersReducedMotion = usePrefersReducedMotion()
+		const [url, setUrl] = useState(asset?.props.src || '')
 
 		const rVideo = useRef<HTMLVideoElement>(null!)
 
+		useEffect(() => {
+			async function retrieveAsset() {
+				// Retrieve a local image from the DB.
+				if (asset && url?.startsWith('asset:')) {
+					const blob = await editor.getAssetBlobFromObjectStore(asset)
+					if (blob) {
+						const imgURL = URL.createObjectURL(blob)
+						setUrl(imgURL)
+					}
+				}
+			}
+			retrieveAsset()
+		}, [url, asset, editor])
+
 		const handlePlay = useCallback<ReactEventHandler<HTMLVideoElement>>(
 			(e) => {
 				const video = e.currentTarget
@@ -145,6 +160,8 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
 			}
 		}, [rVideo, prefersReducedMotion])
 
+		if (url.startsWith('asset:')) return null
+
 		return (
 			<>
 				<HTMLContainer
@@ -157,7 +174,7 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
 				>
 					<div className="tl-counter-scaled">
 						<div className="tl-video-container">
-							{asset?.props.src ? (
+							{url ? (
 								<video
 									ref={rVideo}
 									style={isEditing ? { pointerEvents: 'all' } : undefined}
@@ -178,7 +195,7 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
 									onLoadedData={handleLoadedData}
 									hidden={!isLoaded}
 								>
-									<source src={asset.props.src} />
+									<source src={url} />
 								</video>
 							) : (
 								<BrokenAssetIcon />

packages/validate/src/lib/validation.ts

@@ -981,7 +981,8 @@ export const linkUrl = string.check((value) => {
 	}
 })
 
-const validSrcProtocols = new Set(['http:', 'https:', 'data:'])
+// N.B. asset: is a reference to the local indexedDB object store.
+const validSrcProtocols = new Set(['http:', 'https:', 'data:', 'asset:'])
 
 /**
  * Validates that a valid is a url safe to load as an asset.