assets: make option to transform urls dynamically / LOD (#3827)

this is take #2 of this PR https://github.com/tldraw/tldraw/pull/3764

This continues the idea kicked off in
https://github.com/tldraw/tldraw/pull/3684 to explore LOD and takes it
in a different direction.

Several things here to call out:
- our dotcom version would start to use Cloudflare's image transforms
- we don't rewrite non-image assets 
- we debounce zooming so that we're not swapping out images while
zooming (it creates jank)
- we load different images based on steps of .25 (maybe we want to make
this more, like 0.33). Feels like 0.5 might be a bit too much but we can
play around with it.
- we take into account network connection speed. if you're on 3g, for
example, we have the size of the image.
- dpr is taken into account - in our case, Cloudflare handles it. But if
it wasn't Cloudflare, we could add it to our width equation.
- we use Cloudflare's `fit=scale-down` setting to never scale _up_ an
image.
- we don't swap the image in until we've finished loading it
programatically (to avoid a blank image while it loads)

TODO
- [x] We need to enable Cloudflare's pricing on image transforms btw
@steveruizok 😉 - this won't work quite yet until we do that.


### Change Type

<!-- ❗ Please select a 'Scope' label ❗️ -->

- [x] `sdk` — Changes the tldraw SDK
- [ ] `dotcom` — Changes the tldraw.com web app
- [ ] `docs` — Changes to the documentation, examples, or templates.
- [ ] `vs code` — Changes to the vscode plugin
- [ ] `internal` — Does not affect user-facing stuff

<!-- ❗ Please select a 'Type' label ❗️ -->

- [ ] `bugfix` — Bug fix
- [x] `feature` — New feature
- [ ] `improvement` — Improving existing features
- [ ] `chore` — Updating dependencies, other boring stuff
- [ ] `galaxy brain` — Architectural changes
- [ ] `tests` — Changes to any test code
- [ ] `tools` — Changes to infrastructure, CI, internal scripts,
debugging tools, etc.
- [ ] `dunno` — I don't know


### Test Plan

1. Test images on staging, small, medium, large, mega
2. Test videos on staging

- [x] Unit Tests
- [ ] End to end tests

### Release Notes

- Assets: make option to transform urls dynamically to provide different
sized images on demand.

.github/workflows/deploy.yml

@@ -55,6 +55,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
 
           ASSET_UPLOAD: ${{ vars.ASSET_UPLOAD }}
+          ASSET_BUCKET_ORIGIN: ${{ vars.ASSET_BUCKET_ORIGIN }}
           MULTIPLAYER_SERVER: ${{ vars.MULTIPLAYER_SERVER }}
           SUPABASE_LITE_URL: ${{ vars.SUPABASE_LITE_URL }}
           VERCEL_PROJECT_ID: ${{ vars.VERCEL_DOTCOM_PROJECT_ID }}

apps/dotcom/setupTests.js

@@ -2,6 +2,7 @@ global.crypto ??= new (require('@peculiar/webcrypto').Crypto)()
 
 process.env.MULTIPLAYER_SERVER = 'https://localhost:8787'
 process.env.ASSET_UPLOAD = 'https://localhost:8788'
+process.env.ASSET_BUCKET_ORIGIN = 'https://localhost:8788'
 
 global.TextEncoder = require('util').TextEncoder
 global.TextDecoder = require('util').TextDecoder

apps/dotcom/src/components/LocalEditor.tsx

@@ -19,6 +19,7 @@ import {
 	ViewSubmenu,
 	useActions,
 } from 'tldraw'
+import { resolveAsset } from '../utils/assetHandler'
 import { assetUrls } from '../utils/assetUrls'
 import { createAssetFromUrl } from '../utils/createAssetFromUrl'
 import { DebugMenuItems } from '../utils/migration/DebugMenuItems'
@@ -105,6 +106,7 @@ export function LocalEditor() {
 				overrides={[sharingUiOverrides, fileSystemUiOverrides]}
 				onUiEvent={handleUiEvent}
 				components={components}
+				assetOptions={{ onResolveAsset: resolveAsset }}
 				inferDarkMode
 			>
 				<LocalMigration />

apps/dotcom/src/components/MultiplayerEditor.tsx

@@ -24,6 +24,7 @@ import {
 } from 'tldraw'
 import { useRemoteSyncClient } from '../hooks/useRemoteSyncClient'
 import { UrlStateParams, useUrlState } from '../hooks/useUrlState'
+import { resolveAsset } from '../utils/assetHandler'
 import { assetUrls } from '../utils/assetUrls'
 import { MULTIPLAYER_SERVER } from '../utils/config'
 import { CursorChatMenuItem } from '../utils/context-menu/CursorChatMenuItem'
@@ -158,6 +159,7 @@ export function MultiplayerEditor({
 				initialState={isReadonly ? 'hand' : 'select'}
 				onUiEvent={handleUiEvent}
 				components={components}
+				assetOptions={{ onResolveAsset: resolveAsset }}
 				inferDarkMode
 			>
 				<UrlStateSync />

apps/dotcom/src/hooks/useMultiplayerAssets.ts

@@ -54,6 +54,7 @@ export function useMultiplayerAssets(assetUploaderUrl: string) {
 					src: url,
 					w: size.w,
 					h: size.h,
+					fileSize: file.size,
 					mimeType: file.type,
 					isAnimated,
 				},

apps/dotcom/src/utils/assetHandler.test.ts

@@ -0,0 +1,202 @@
+import { TLAsset } from 'tldraw'
+import { resolveAsset } from './assetHandler'
+
+const FILE_SIZE = 1024 * 1024 * 2
+
+describe('resolveAsset', () => {
+	it('should return null if the asset is null', async () => {
+		expect(
+			await resolveAsset(null, {
+				screenScale: -1,
+				steppedScreenScale: 1,
+				dpr: 1,
+				networkEffectiveType: '4g',
+			})
+		).toBe(null)
+	})
+
+	it('should return null if the asset is undefined', async () => {
+		expect(
+			await resolveAsset(undefined, {
+				screenScale: -1,
+				steppedScreenScale: 1,
+				dpr: 1,
+				networkEffectiveType: '4g',
+			})
+		).toBe(null)
+	})
+
+	it('should return null if the asset has no src', async () => {
+		const asset = { type: 'image', props: { w: 100, fileSize: FILE_SIZE } }
+		expect(
+			await resolveAsset(asset as TLAsset, {
+				screenScale: -1,
+				steppedScreenScale: 1,
+				dpr: 1,
+				networkEffectiveType: '4g',
+			})
+		).toBe(null)
+	})
+
+	it('should return the original src for video types', async () => {
+		const asset = {
+			type: 'video',
+			props: { src: 'http://example.com/video.mp4', fileSize: FILE_SIZE },
+		}
+		expect(
+			await resolveAsset(asset as TLAsset, {
+				screenScale: -1,
+				steppedScreenScale: 1,
+				dpr: 1,
+				networkEffectiveType: '4g',
+			})
+		).toBe('http://example.com/video.mp4')
+	})
+
+	it('should return the original src if it does not start with http or https', async () => {
+		const asset = { type: 'image', props: { src: 'data:somedata', w: 100, fileSize: FILE_SIZE } }
+		expect(
+			await resolveAsset(asset as TLAsset, {
+				screenScale: -1,
+				steppedScreenScale: 1,
+				dpr: 1,
+				networkEffectiveType: '4g',
+			})
+		).toBe('data:somedata')
+	})
+
+	it('should return the original src if it is animated', async () => {
+		const asset = {
+			type: 'image',
+			props: {
+				src: 'http://example.com/animated.gif',
+				mimeType: 'image/gif',
+				w: 100,
+				fileSize: FILE_SIZE,
+			},
+		}
+		expect(
+			await resolveAsset(asset as TLAsset, {
+				screenScale: -1,
+				steppedScreenScale: 1,
+				dpr: 1,
+				networkEffectiveType: '4g',
+			})
+		).toBe('http://example.com/animated.gif')
+	})
+
+	it('should return the original src if it is under a certain file size', async () => {
+		const asset = {
+			type: 'image',
+			props: { src: 'http://example.com/small.png', w: 100, fileSize: 1024 * 1024 },
+		}
+		expect(
+			await resolveAsset(asset as TLAsset, {
+				screenScale: -1,
+				steppedScreenScale: 1,
+				dpr: 1,
+				networkEffectiveType: '4g',
+			})
+		).toBe('http://example.com/small.png')
+	})
+
+	it("should return null if the asset type is not 'image'", async () => {
+		const asset = {
+			type: 'document',
+			props: { src: 'http://example.com/doc.pdf', w: 100, fileSize: FILE_SIZE },
+		}
+		expect(
+			await resolveAsset(asset as TLAsset, {
+				screenScale: -1,
+				steppedScreenScale: 1,
+				dpr: 1,
+				networkEffectiveType: '4g',
+			})
+		).toBe(null)
+	})
+
+	it('should handle if network compensation is not available and zoom correctly', async () => {
+		const asset = {
+			type: 'image',
+			props: { src: 'http://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
+		}
+		expect(
+			await resolveAsset(asset as TLAsset, {
+				screenScale: -1,
+				steppedScreenScale: 0.5,
+				dpr: 2,
+				networkEffectiveType: null,
+			})
+		).toBe(
+			'https://localhost:8788/cdn-cgi/image/width=50,dpr=2,fit=scale-down,quality=92/http://example.com/image.jpg'
+		)
+	})
+
+	it('should handle network compensation and zoom correctly', async () => {
+		const asset = {
+			type: 'image',
+			props: { src: 'http://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
+		}
+		expect(
+			await resolveAsset(asset as TLAsset, {
+				screenScale: -1,
+				steppedScreenScale: 0.5,
+				dpr: 2,
+				networkEffectiveType: '3g',
+			})
+		).toBe(
+			'https://localhost:8788/cdn-cgi/image/width=25,dpr=2,fit=scale-down,quality=92/http://example.com/image.jpg'
+		)
+	})
+
+	it('should round zoom to powers of 2', async () => {
+		const asset = {
+			type: 'image',
+			props: { src: 'https://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
+		}
+		expect(
+			await resolveAsset(asset as TLAsset, {
+				screenScale: -1,
+				steppedScreenScale: 4,
+				dpr: 1,
+				networkEffectiveType: '4g',
+			})
+		).toBe(
+			'https://localhost:8788/cdn-cgi/image/width=400,dpr=1,fit=scale-down,quality=92/https://example.com/image.jpg'
+		)
+	})
+
+	it('should round zoom to the nearest 0.25 and apply network compensation', async () => {
+		const asset = {
+			type: 'image',
+			props: { src: 'https://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
+		}
+		expect(
+			await resolveAsset(asset as TLAsset, {
+				screenScale: -1,
+				steppedScreenScale: 0.5,
+				dpr: 1,
+				networkEffectiveType: '2g',
+			})
+		).toBe(
+			'https://localhost:8788/cdn-cgi/image/width=25,dpr=1,fit=scale-down,quality=92/https://example.com/image.jpg'
+		)
+	})
+
+	it('should set zoom to a minimum of 0.25 if zoom is below 0.25', async () => {
+		const asset = {
+			type: 'image',
+			props: { src: 'https://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
+		}
+		expect(
+			await resolveAsset(asset as TLAsset, {
+				screenScale: -1,
+				steppedScreenScale: 0.25,
+				dpr: 1,
+				networkEffectiveType: '4g',
+			})
+		).toBe(
+			'https://localhost:8788/cdn-cgi/image/width=25,dpr=1,fit=scale-down,quality=92/https://example.com/image.jpg'
+		)
+	})
+})

apps/dotcom/src/utils/assetHandler.ts

@@ -0,0 +1,40 @@
+import { AssetContextProps, MediaHelpers, TLAsset } from 'tldraw'
+import { ASSET_BUCKET_ORIGIN, ASSET_UPLOADER_URL } from './config'
+
+export async function resolveAsset(asset: TLAsset | null | undefined, context: AssetContextProps) {
+	if (!asset || !asset.props.src) return null
+
+	// We don't deal with videos at the moment.
+	if (asset.type === 'video') return asset.props.src
+
+	// Assert it's an image to make TS happy.
+	if (asset.type !== 'image') return null
+
+	// Don't try to transform data: URLs, yikes.
+	if (!asset.props.src.startsWith('http:') && !asset.props.src.startsWith('https:'))
+		return asset.props.src
+
+	// Don't try to transform animated images.
+	if (MediaHelpers.isAnimatedImageType(asset?.props.mimeType) || asset.props.isAnimated)
+		return asset.props.src
+
+	// Assets that are under a certain file size aren't worth transforming (and incurring cost).
+	if (asset.props.fileSize === -1 || asset.props.fileSize < 1024 * 1024 * 1.5 /* 1.5 MB */)
+		return asset.props.src
+
+	// N.B. navigator.connection is only available in certain browsers (mainly Blink-based browsers)
+	// 4g is as high the 'effectiveType' goes and we can pick a lower effective image quality for slower connections.
+	const networkCompensation =
+		!context.networkEffectiveType || context.networkEffectiveType === '4g' ? 1 : 0.5
+
+	const width = Math.ceil(asset.props.w * context.steppedScreenScale * networkCompensation)
+
+	if (process.env.NODE_ENV === 'development') {
+		return asset.props.src
+	}
+
+	// On preview, builds the origin for the asset won't be the right one for the Cloudflare transform.
+	const src = asset.props.src.replace(ASSET_UPLOADER_URL, ASSET_BUCKET_ORIGIN)
+
+	return `${ASSET_BUCKET_ORIGIN}/cdn-cgi/image/width=${width},dpr=${context.dpr},fit=scale-down,quality=92/${src}`
+}

apps/dotcom/src/utils/config.ts

@@ -6,8 +6,14 @@ if (!process.env.ASSET_UPLOAD) {
 	throw new Error('Missing ASSET_UPLOAD env var')
 }
 
+if (!process.env.ASSET_BUCKET_ORIGIN) {
+	throw new Error('Missing ASSET_BUCKET_ORIGIN env var')
+}
+
 export const ASSET_UPLOADER_URL: string = process.env.ASSET_UPLOAD
 
+export const ASSET_BUCKET_ORIGIN: string = process.env.ASSET_BUCKET_ORIGIN
+
 export const CONTROL_SERVER: string =
 	process.env.NEXT_PUBLIC_CONTROL_SERVER || 'http://localhost:3001'
 

apps/dotcom/src/utils/createAssetFromFile.ts

@@ -53,6 +53,7 @@ export async function createAssetFromFile({ file }: { type: 'file'; file: File }
 			w: size.w,
 			h: size.h,
 			mimeType: file.type,
+			fileSize: file.size,
 			isAnimated,
 		},
 		meta: {},

apps/dotcom/vite.config.ts

@@ -46,6 +46,11 @@ export default defineConfig((env) => ({
 		),
 		'process.env.MULTIPLAYER_SERVER': urlOrLocalFallback(env.mode, getMultiplayerServerURL(), 8787),
 		'process.env.ASSET_UPLOAD': urlOrLocalFallback(env.mode, process.env.ASSET_UPLOAD, 8788),
+		'process.env.ASSET_BUCKET_ORIGIN': urlOrLocalFallback(
+			env.mode,
+			process.env.ASSET_BUCKET_ORIGIN,
+			8788
+		),
 		'process.env.TLDRAW_ENV': JSON.stringify(process.env.TLDRAW_ENV ?? 'development'),
 		// Fall back to staging DSN for local develeopment, although you still need to
 		// modify the env check in 'sentry.client.config.ts' to get it reporting errors

apps/examples/src/examples/hosted-images/HostedImagesExample.tsx

@@ -58,6 +58,7 @@ export default function HostedImagesExample() {
 					src: url,
 					w: size.w,
 					h: size.h,
+					fileSize: file.size,
 					mimeType: file.type,
 					isAnimated,
 				},

apps/examples/src/examples/image-annotator/ImageAnnotationEditor.tsx

@@ -48,6 +48,7 @@ export function ImageAnnotationEditor({
 				props: {
 					w: image.width,
 					h: image.height,
+					fileSize: -1,
 					mimeType: image.type,
 					src: image.src,
 					name: 'image',

apps/examples/src/examples/local-images/LocalImagesExample.tsx

@@ -21,6 +21,7 @@ export default function LocalImagesExample() {
 					src: '/tldraw.png', // You could also use a base64 encoded string here
 					w: imageWidth,
 					h: imageHeight,
+					fileSize: -1,
 					mimeType: 'image/png',
 					isAnimated: false,
 				},

apps/examples/src/examples/pdf-editor/PdfEditor.tsx

@@ -43,6 +43,7 @@ export function PdfEditor({ pdf }: { pdf: Pdf }) {
 						props: {
 							w: page.bounds.w,
 							h: page.bounds.h,
+							fileSize: -1,
 							mimeType: 'image/png',
 							src: page.src,
 							name: 'page',

packages/editor/api-report.md

@@ -85,6 +85,7 @@ import { useComputed } from '@tldraw/state';
 import { useQuickReactor } from '@tldraw/state';
 import { useReactor } from '@tldraw/state';
 import { useValue } from '@tldraw/state';
+import { useValueDebounced } from '@tldraw/state';
 import { VecModel } from '@tldraw/tlschema';
 import { whyAmIRunning } from '@tldraw/state';
 
@@ -144,6 +145,18 @@ export class Arc2d extends Geometry2d {
 // @public
 export function areAnglesCompatible(a: number, b: number): boolean;
 
+// @public (undocumented)
+export interface AssetContextProps {
+    // (undocumented)
+    dpr: number;
+    // (undocumented)
+    networkEffectiveType: null | string;
+    // (undocumented)
+    screenScale: number;
+    // (undocumented)
+    steppedScreenScale: number;
+}
+
 export { Atom }
 
 export { atom }
@@ -765,7 +778,7 @@ export class Edge2d extends Geometry2d {
 
 // @public (undocumented)
 export class Editor extends EventEmitter<TLEventMap> {
-    constructor({ store, user, shapeUtils, bindingUtils, tools, getContainer, cameraOptions, initialState, autoFocus, inferDarkMode, options, }: TLEditorOptions);
+    constructor({ store, user, shapeUtils, bindingUtils, tools, getContainer, cameraOptions, assetOptions, initialState, autoFocus, inferDarkMode, options, }: TLEditorOptions);
     addOpenMenu(id: string): this;
     alignShapes(shapes: TLShape[] | TLShapeId[], operation: 'bottom' | 'center-horizontal' | 'center-vertical' | 'left' | 'right' | 'top'): this;
     animateShape(partial: null | TLShapePartial | undefined, opts?: Partial<{
@@ -1114,6 +1127,10 @@ export class Editor extends EventEmitter<TLEventMap> {
     reparentShapes(shapes: TLShape[] | TLShapeId[], parentId: TLParentId, insertIndex?: IndexKey): this;
     resetZoom(point?: Vec, opts?: TLCameraMoveOptions): this;
     resizeShape(shape: TLShape | TLShapeId, scale: VecLike, options?: TLResizeShapeOptions): this;
+    // (undocumented)
+    resolveAssetUrl(assetId: null | TLAssetId, context: {
+        screenScale: number;
+    }): Promise<null | string>;
     readonly root: StateNode;
     rotateShapesBy(shapes: TLShape[] | TLShapeId[], delta: number): this;
     screenToPage(point: VecLike): Vec;
@@ -2306,6 +2323,12 @@ export type TLAnyBindingUtilConstructor = TLBindingUtilConstructor<any>;
 // @public (undocumented)
 export type TLAnyShapeUtilConstructor = TLShapeUtilConstructor<any>;
 
+// @public (undocumented)
+export interface TLAssetOptions {
+    // (undocumented)
+    onResolveAsset: (asset: null | TLAsset | undefined, ctx: AssetContextProps) => Promise<null | string>;
+}
+
 // @public (undocumented)
 export type TLBaseBoxShape = TLBaseShape<string, {
     h: number;
@@ -2478,6 +2501,7 @@ export const TldrawEditor: React_2.NamedExoticComponent<TldrawEditorProps>;
 
 // @public
 export interface TldrawEditorBaseProps {
+    assetOptions?: Partial<TLAssetOptions>;
     autoFocus?: boolean;
     bindingUtils?: readonly TLAnyBindingUtilConstructor[];
     cameraOptions?: Partial<TLCameraOptions>;
@@ -2622,6 +2646,7 @@ export interface TLEditorComponents {
 
 // @public (undocumented)
 export interface TLEditorOptions {
+    assetOptions?: Partial<TLAssetOptions>;
     autoFocus?: boolean;
     bindingUtils: readonly TLBindingUtilConstructor<TLUnknownBinding>[];
     cameraOptions?: Partial<TLCameraOptions>;
@@ -3453,6 +3478,8 @@ export function useTransform(ref: React.RefObject<HTMLElement | SVGElement>, x?:
 
 export { useValue }
 
+export { useValueDebounced }
+
 // @public (undocumented)
 export class Vec {
     constructor(x?: number, y?: number, z?: number);

packages/editor/src/index.ts

@@ -28,6 +28,7 @@ export {
 	useQuickReactor,
 	useReactor,
 	useValue,
+	useValueDebounced,
 	whyAmIRunning,
 	type Atom,
 	type Signal,
@@ -239,8 +240,10 @@ export {
 	type TLHistoryMark,
 } from './lib/editor/types/history-types'
 export {
+	type AssetContextProps,
 	type OptionalKeys,
 	type RequiredKeys,
+	type TLAssetOptions,
 	type TLCameraConstraints,
 	type TLCameraMoveOptions,
 	type TLCameraOptions,

packages/editor/src/lib/TldrawEditor.tsx

@@ -20,7 +20,7 @@ import { TLAnyBindingUtilConstructor } from './config/defaultBindings'
 import { TLAnyShapeUtilConstructor } from './config/defaultShapes'
 import { Editor } from './editor/Editor'
 import { TLStateNodeConstructor } from './editor/tools/StateNode'
-import { TLCameraOptions } from './editor/types/misc-types'
+import { TLAssetOptions, TLCameraOptions } from './editor/types/misc-types'
 import { ContainerProvider, useContainer } from './hooks/useContainer'
 import { useCursor } from './hooks/useCursor'
 import { useDarkMode } from './hooks/useDarkMode'
@@ -127,6 +127,11 @@ export interface TldrawEditorBaseProps {
 	 */
 	cameraOptions?: Partial<TLCameraOptions>
 
+	/**
+	 * Asset options for the editor.
+	 */
+	assetOptions?: Partial<TLAssetOptions>
+
 	/**
 	 * Options for the editor.
 	 */
@@ -300,6 +305,7 @@ function TldrawEditorWithReadyStore({
 	autoFocus = true,
 	inferDarkMode,
 	cameraOptions,
+	assetOptions,
 	options,
 }: Required<
 	TldrawEditorProps & {
@@ -325,6 +331,7 @@ function TldrawEditorWithReadyStore({
 			autoFocus: initialAutoFocus,
 			inferDarkMode,
 			cameraOptions,
+			assetOptions,
 			options,
 		})
 		setEditor(editor)
@@ -343,6 +350,7 @@ function TldrawEditorWithReadyStore({
 		initialAutoFocus,
 		inferDarkMode,
 		cameraOptions,
+		assetOptions,
 		options,
 	])
 

packages/editor/src/lib/constants.ts

@@ -1,4 +1,4 @@
-import { TLCameraOptions } from './editor/types/misc-types'
+import { TLAssetOptions, TLCameraOptions } from './editor/types/misc-types'
 import { EASINGS } from './primitives/easings'
 
 /** @internal */
@@ -10,6 +10,11 @@ export const DEFAULT_CAMERA_OPTIONS: TLCameraOptions = {
 	zoomSteps: [0.1, 0.25, 0.5, 1, 2, 4, 8],
 }
 
+/** @internal */
+export const DEFAULT_ASSET_OPTIONS: TLAssetOptions = {
+	onResolveAsset: async (asset) => asset?.props.src || '',
+}
+
 /** @internal */
 export const DEFAULT_ANIMATION_OPTIONS = {
 	duration: 0,

packages/editor/src/lib/editor/Editor.ts

@@ -85,6 +85,7 @@ import { checkBindings } from '../config/defaultBindings'
 import { checkShapesAndAddCore } from '../config/defaultShapes'
 import {
 	DEFAULT_ANIMATION_OPTIONS,
+	DEFAULT_ASSET_OPTIONS,
 	DEFAULT_CAMERA_OPTIONS,
 	INTERNAL_POINTER_IDS,
 	LEFT_MOUSE_BUTTON,
@@ -144,6 +145,7 @@ import { TLHistoryBatchOptions } from './types/history-types'
 import {
 	OptionalKeys,
 	RequiredKeys,
+	TLAssetOptions,
 	TLCameraMoveOptions,
 	TLCameraOptions,
 	TLSvgOptions,
@@ -207,7 +209,10 @@ export interface TLEditorOptions {
 	 * Options for the editor's camera.
 	 */
 	cameraOptions?: Partial<TLCameraOptions>
-
+	/**
+	 * Options for the editor's assets.
+	 */
+	assetOptions?: Partial<TLAssetOptions>
 	options?: Partial<TldrawOptions>
 }
 
@@ -221,6 +226,7 @@ export class Editor extends EventEmitter<TLEventMap> {
 		tools,
 		getContainer,
 		cameraOptions,
+		assetOptions,
 		initialState,
 		autoFocus,
 		inferDarkMode,
@@ -246,6 +252,8 @@ export class Editor extends EventEmitter<TLEventMap> {
 
 		this._cameraOptions.set({ ...DEFAULT_CAMERA_OPTIONS, ...cameraOptions })
 
+		this._assetOptions.set({ ...DEFAULT_ASSET_OPTIONS, ...assetOptions })
+
 		this.user = new UserPreferencesManager(user ?? createTLUser(), inferDarkMode ?? false)
 
 		this.getContainer = getContainer ?? (() => document.body)
@@ -3815,6 +3823,8 @@ export class Editor extends EventEmitter<TLEventMap> {
 
 	/* --------------------- Assets --------------------- */
 
+	private _assetOptions = atom('asset options', DEFAULT_ASSET_OPTIONS)
+
 	/** @internal */
 	@computed private _getAllAssetsQuery() {
 		return this.store.query.records('asset')
@@ -3915,6 +3925,29 @@ export class Editor extends EventEmitter<TLEventMap> {
 		return this.store.get(typeof asset === 'string' ? asset : asset.id) as TLAsset | undefined
 	}
 
+	async resolveAssetUrl(
+		assetId: TLAssetId | null,
+		context: { screenScale: number }
+	): Promise<string | null> {
+		if (!assetId) return ''
+		const asset = this.getAsset(assetId)
+		if (!asset) return ''
+
+		// We only look at the zoom level at powers of 2.
+		const zoomStepFunction = (zoom: number) => Math.pow(2, Math.ceil(Math.log2(zoom)))
+		const steppedScreenScale = Math.max(0.125, zoomStepFunction(context.screenScale))
+		const networkEffectiveType: string | null =
+			'connection' in navigator ? (navigator as any).connection.effectiveType : null
+		const dpr = this.getInstanceState().devicePixelRatio
+
+		return await this._assetOptions.get().onResolveAsset(asset!, {
+			screenScale: context.screenScale,
+			steppedScreenScale,
+			dpr,
+			networkEffectiveType,
+		})
+	}
+
 	/* --------------------- Shapes --------------------- */
 
 	@computed

packages/editor/src/lib/editor/types/misc-types.ts

@@ -1,4 +1,4 @@
-import { BoxModel } from '@tldraw/tlschema'
+import { BoxModel, TLAsset } from '@tldraw/tlschema'
 import { Box } from '../../primitives/Box'
 import { VecLike } from '../../primitives/Vec'
 
@@ -55,6 +55,22 @@ export interface TLCameraOptions {
 	constraints?: TLCameraConstraints
 }
 
+/** @public */
+export interface AssetContextProps {
+	screenScale: number
+	steppedScreenScale: number
+	dpr: number
+	networkEffectiveType: string | null
+}
+
+/** @public */
+export interface TLAssetOptions {
+	onResolveAsset: (
+		asset: TLAsset | null | undefined,
+		ctx: AssetContextProps
+	) => Promise<string | null>
+}
+
 /** @public */
 export interface TLCameraConstraints {
 	/** The bounds (in page space) of the constrained space */

packages/state/api-report.md

@@ -207,6 +207,9 @@ export function useValue<Value>(value: Signal<Value>): Value;
 // @public (undocumented)
 export function useValue<Value>(name: string, fn: () => Value, deps: unknown[]): Value;
 
+// @public
+export function useValueDebounced<Value>(name: string, fn: () => Value, deps: unknown[], ms: number): Value;
+
 // @public
 export function whyAmIRunning(): void;
 

packages/state/src/lib/react/index.ts

@@ -5,3 +5,4 @@ export { useQuickReactor } from './useQuickReactor'
 export { useReactor } from './useReactor'
 export { useStateTracking } from './useStateTracking'
 export { useValue } from './useValue'
+export { useValueDebounced } from './useValueDebounced'

packages/state/src/lib/react/useValueDebounced.ts

@@ -0,0 +1,33 @@
+/* eslint-disable prefer-rest-params */
+import { useEffect, useState } from 'react'
+import { useValue } from './useValue'
+
+/**
+ * Extracts the value from a signal and subscribes to it, debouncing the value by the given number of milliseconds.
+ *
+ * @see [[useValue]] for more information.
+ *
+ * @public
+ */
+export function useValueDebounced<Value>(
+	name: string,
+	fn: () => Value,
+	deps: unknown[],
+	ms: number
+): Value
+/** @public */
+export function useValueDebounced<Value>(): Value {
+	const args = [...arguments].slice(0, -1) as Parameters<typeof useValue>
+	const ms = arguments[arguments.length - 1] as number
+	const value = useValue(...args) as Value
+	const [debouncedValue, setDebouncedValue] = useState<Value>(value)
+
+	useEffect(() => {
+		const timer = setTimeout(() => {
+			setDebouncedValue(value)
+		}, ms)
+		return () => clearTimeout(timer)
+	}, [value, ms])
+
+	return debouncedValue
+}

packages/tldraw/src/lib/defaultExternalContentHandlers.ts

@@ -23,7 +23,7 @@ import {
 import { FONT_FAMILIES, FONT_SIZES, TEXT_PROPS } from './shapes/shared/default-shape-constants'
 import { TLUiToastsContextType } from './ui/context/toasts'
 import { useTranslation } from './ui/hooks/useTranslation/useTranslation'
-import { containBoxSize, downsizeImage } from './utils/assets/assets'
+import { containBoxSize } from './utils/assets/assets'
 import { getEmbedInfo } from './utils/embeds/embeds'
 import { cleanupText, isRightToLeftLanguage } from './utils/text/text'
 
@@ -82,14 +82,6 @@ export function registerDefaultExternalContentHandlers(
 			}
 		}
 
-		// Always rescale the image
-		if (!isAnimated && MediaHelpers.isStaticImageType(file.type)) {
-			file = await downsizeImage(file, size.w, size.h, {
-				type: file.type,
-				quality: 0.92,
-			})
-		}
-
 		const assetId: TLAssetId = AssetRecordType.createId(hash)
 
 		const asset = AssetRecordType.create({
@@ -101,6 +93,7 @@ export function registerDefaultExternalContentHandlers(
 				src: await FileHelpers.blobToDataUrl(file),
 				w: size.w,
 				h: size.h,
+				fileSize: file.size,
 				mimeType: file.type,
 				isAnimated,
 			},

packages/tldraw/src/lib/shapes/image/ImageShapeUtil.tsx

@@ -18,6 +18,7 @@ import {
 import { useEffect, useState } from 'react'
 import { BrokenAssetIcon } from '../shared/BrokenAssetIcon'
 import { HyperlinkButton } from '../shared/HyperlinkButton'
+import { useAsset } from '../shared/useAsset'
 import { usePrefersReducedMotion } from '../shared/usePrefersReducedMotion'
 
 async function getDataURIFromURL(url: string): Promise<string> {
@@ -61,15 +62,35 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 		const isCropping = this.editor.getCroppingShapeId() === shape.id
 		const prefersReducedMotion = usePrefersReducedMotion()
 		const [staticFrameSrc, setStaticFrameSrc] = useState('')
-
-		const asset = shape.props.assetId ? this.editor.getAsset(shape.props.assetId) : undefined
-
+		const [loadedSrc, setLoadedSrc] = useState('')
 		const isSelected = shape.id === this.editor.getOnlySelectedShapeId()
+		const { asset, url } = useAsset(shape.props.assetId, shape.props.w)
 
 		useEffect(() => {
-			if (asset?.props.src && this.isAnimated(shape)) {
+			// If an image is not animated (that's handled below), then we preload the image
+			// because we might have different source urls for different zoom levels.
+			// Preloading the image ensures that the browser caches the image and doesn't
+			// cause visual flickering when the image is loaded.
+			if (url && !this.isAnimated(shape)) {
+				let cancelled = false
+				if (!url) return
+
+				const image = Image()
+				image.onload = () => {
+					if (cancelled) return
+					setLoadedSrc(url)
+				}
+				image.src = url
+
+				return () => {
+					cancelled = true
+				}
+			}
+		}, [url, shape])
+
+		useEffect(() => {
+			if (url && this.isAnimated(shape)) {
 				let cancelled = false
-				const url = asset.props.src
 				if (!url) return
 
 				const image = Image()
@@ -85,6 +106,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 
 					ctx.drawImage(image, 0, 0)
 					setStaticFrameSrc(canvas.toDataURL())
+					setLoadedSrc(url)
 				}
 				image.crossOrigin = 'anonymous'
 				image.referrerPolicy = 'strict-origin-when-cross-origin'
@@ -94,7 +116,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 					cancelled = true
 				}
 			}
-		}, [prefersReducedMotion, asset?.props, shape])
+		}, [prefersReducedMotion, url, shape])
 
 		if (asset?.type === 'bookmark') {
 			throw Error("Bookmark assets can't be rendered as images")
@@ -108,7 +130,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 
 		const containerStyle = getCroppedContainerStyle(shape)
 
-		if (!asset?.props.src) {
+		if (!url) {
 			return (
 				<HTMLContainer
 					id={shape.id}
@@ -141,7 +163,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 							style={{
 								opacity: 0.1,
 								backgroundImage: `url(${
-									!shape.props.playing || reduceMotion ? staticFrameSrc : asset.props.src
+									!shape.props.playing || reduceMotion ? staticFrameSrc : loadedSrc
 								})`,
 							}}
 							draggable={false}
@@ -157,7 +179,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 							className="tl-image"
 							style={{
 								backgroundImage: `url(${
-									!shape.props.playing || reduceMotion ? staticFrameSrc : asset.props.src
+									!shape.props.playing || reduceMotion ? staticFrameSrc : loadedSrc
 								})`,
 							}}
 							draggable={false}

packages/tldraw/src/lib/shapes/shared/useAsset.ts

@@ -0,0 +1,31 @@
+import { TLAssetId, useEditor, useValueDebounced } from '@tldraw/editor'
+import { useEffect, useState } from 'react'
+
+/** @internal */
+export function useAsset(assetId: TLAssetId | null, width: number) {
+	const editor = useEditor()
+	const [url, setUrl] = useState<string | null>(null)
+	const asset = assetId ? editor.getAsset(assetId) : null
+
+	const shapeScale = asset && 'w' in asset.props ? width / asset.props.w : 1
+	// We debounce the zoom level to reduce the number of times we fetch a new image and,
+	// more importantly, to not cause zooming in and out to feel janky.
+	const debouncedScreenScale = useValueDebounced(
+		'zoom level',
+		() => editor.getZoomLevel() * shapeScale,
+		[editor, shapeScale],
+		500
+	)
+
+	useEffect(() => {
+		async function resolve() {
+			const resolvedUrl = await editor.resolveAssetUrl(assetId, {
+				screenScale: debouncedScreenScale,
+			})
+			setUrl(resolvedUrl)
+		}
+		resolve()
+	}, [assetId, debouncedScreenScale, editor])
+
+	return { asset, url }
+}

packages/tldraw/src/lib/shapes/video/VideoShapeUtil.tsx

@@ -11,6 +11,7 @@ import {
 import { ReactEventHandler, useCallback, useEffect, useRef, useState } from 'react'
 import { BrokenAssetIcon } from '../shared/BrokenAssetIcon'
 import { HyperlinkButton } from '../shared/HyperlinkButton'
+import { useAsset } from '../shared/useAsset'
 import { usePrefersReducedMotion } from '../shared/usePrefersReducedMotion'
 
 /** @public */
@@ -36,7 +37,7 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
 	component(shape: TLVideoShape) {
 		const { editor } = this
 		const showControls = editor.getShapeGeometry(shape).bounds.w * editor.getZoomLevel() >= 110
-		const asset = shape.props.assetId ? editor.getAsset(shape.props.assetId) : null
+		const { asset, url } = useAsset(shape.props.assetId, shape.props.w)
 		const { time, playing } = shape.props
 		const isEditing = useIsEditing(shape.id)
 		const prefersReducedMotion = usePrefersReducedMotion()
@@ -157,7 +158,7 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
 				>
 					<div className="tl-counter-scaled">
 						<div className="tl-video-container">
-							{asset?.props.src ? (
+							{url ? (
 								<video
 									ref={rVideo}
 									style={isEditing ? { pointerEvents: 'all' } : undefined}
@@ -178,7 +179,7 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
 									onLoadedData={handleLoadedData}
 									hidden={!isLoaded}
 								>
-									<source src={asset.props.src} />
+									<source src={url} />
 								</video>
 							) : (
 								<BrokenAssetIcon />

packages/tldraw/src/lib/ui/hooks/clipboard/pasteExcalidrawContent.ts

@@ -290,6 +290,7 @@ export async function pasteExcalidrawContent(editor: Editor, clipboard: any, poi
 					props: {
 						w: element.width,
 						h: element.height,
+						fileSize: file.size,
 						name: element.id ?? 'Untitled',
 						isAnimated: false,
 						mimeType: file.mimeType,

packages/tldraw/src/lib/utils/tldr/buildFromV1Document.ts

@@ -68,6 +68,7 @@ export function buildFromV1Document(editor: Editor, _document: unknown) {
 						props: {
 							w: coerceDimension(v1Asset.size[0]),
 							h: coerceDimension(v1Asset.size[1]),
+							fileSize: -1,
 							name: v1Asset.fileName ?? 'Untitled',
 							isAnimated: false,
 							mimeType: null,
@@ -91,6 +92,7 @@ export function buildFromV1Document(editor: Editor, _document: unknown) {
 								props: {
 									w: coerceDimension(v1Asset.size[0]),
 									h: coerceDimension(v1Asset.size[1]),
+									fileSize: -1,
 									name: v1Asset.fileName ?? 'Untitled',
 									isAnimated: true,
 									mimeType: null,

packages/tldraw/src/test/Editor.test.tsx

@@ -541,6 +541,7 @@ describe('snapshots', () => {
 				props: {
 					w: 1200,
 					h: 800,
+					fileSize: -1,
 					name: '',
 					isAnimated: false,
 					mimeType: 'png',

packages/tlschema/api-report.md

@@ -1187,6 +1187,7 @@ export type TLHighlightShapeProps = RecordPropsType<typeof highlightShapeProps>;
 
 // @public
 export type TLImageAsset = TLBaseAsset<'image', {
+    fileSize: number;
     h: number;
     isAnimated: boolean;
     mimeType: null | string;
@@ -1486,6 +1487,7 @@ export type TLUnknownShape = TLBaseShape<string, object>;
 
 // @public
 export type TLVideoAsset = TLBaseAsset<'video', {
+    fileSize: number;
     h: number;
     isAnimated: boolean;
     mimeType: null | string;

packages/tlschema/src/assets/TLImageAsset.ts

@@ -12,6 +12,7 @@ export type TLImageAsset = TLBaseAsset<
 	{
 		w: number
 		h: number
+		fileSize: number
 		name: string
 		isAnimated: boolean
 		mimeType: string | null
@@ -25,6 +26,7 @@ export const imageAssetValidator: T.Validator<TLImageAsset> = createAssetValidat
 	T.object({
 		w: T.number,
 		h: T.number,
+		fileSize: T.number,
 		name: T.string,
 		isAnimated: T.boolean,
 		mimeType: T.string.nullable(),
@@ -36,6 +38,7 @@ const Versions = createMigrationIds('com.tldraw.asset.image', {
 	AddIsAnimated: 1,
 	RenameWidthHeight: 2,
 	MakeUrlsValid: 3,
+	AddFileSize: 4,
 } as const)
 
 export { Versions as imageAssetVersions }
@@ -81,5 +84,14 @@ export const imageAssetMigrations = createRecordMigrationSequence({
 				// noop
 			},
 		},
+		{
+			id: Versions.AddFileSize,
+			up: (asset: any) => {
+				asset.props.fileSize = -1
+			},
+			down: (asset: any) => {
+				delete asset.props.fileSize
+			},
+		},
 	],
 })

packages/tlschema/src/assets/TLVideoAsset.ts

@@ -12,6 +12,7 @@ export type TLVideoAsset = TLBaseAsset<
 	{
 		w: number
 		h: number
+		fileSize: number
 		name: string
 		isAnimated: boolean
 		mimeType: string | null
@@ -25,6 +26,7 @@ export const videoAssetValidator: T.Validator<TLVideoAsset> = createAssetValidat
 	T.object({
 		w: T.number,
 		h: T.number,
+		fileSize: T.number,
 		name: T.string,
 		isAnimated: T.boolean,
 		mimeType: T.string.nullable(),
@@ -36,6 +38,7 @@ const Versions = createMigrationIds('com.tldraw.asset.video', {
 	AddIsAnimated: 1,
 	RenameWidthHeight: 2,
 	MakeUrlsValid: 3,
+	AddFileSize: 4,
 } as const)
 
 export { Versions as videoAssetVersions }
@@ -81,5 +84,14 @@ export const videoAssetMigrations = createRecordMigrationSequence({
 				// noop
 			},
 		},
+		{
+			id: Versions.AddFileSize,
+			up: (asset: any) => {
+				asset.props.fileSize = -1
+			},
+			down: (asset: any) => {
+				delete asset.props.fileSize
+			},
+		},
 	],
 })

packages/tlschema/src/migrations.test.ts

@@ -98,6 +98,78 @@ describe('TLImageAsset AddIsAnimated', () => {
 	})
 })
 
+describe('TLVideoAsset AddFileSize', () => {
+	const oldAsset = {
+		id: '1',
+		type: 'video',
+		props: {
+			src: 'https://www.youtube.com/watch?v=1',
+			name: 'video',
+			width: 100,
+			height: 100,
+			mimeType: 'video/mp4',
+		},
+	}
+
+	const newAsset = {
+		id: '1',
+		type: 'video',
+		props: {
+			src: 'https://www.youtube.com/watch?v=1',
+			name: 'video',
+			width: 100,
+			height: 100,
+			mimeType: 'video/mp4',
+			fileSize: -1,
+		},
+	}
+
+	const { up, down } = getTestMigration(videoAssetVersions.AddFileSize)
+
+	test('up works as expected', () => {
+		expect(up(oldAsset)).toEqual(newAsset)
+	})
+	test('down works as expected', () => {
+		expect(down(newAsset)).toEqual(oldAsset)
+	})
+})
+
+describe('TLImageAsset AddFileSize', () => {
+	const oldAsset = {
+		id: '1',
+		type: 'image',
+		props: {
+			src: 'https://www.youtube.com/watch?v=1',
+			name: 'image',
+			width: 100,
+			height: 100,
+			mimeType: 'image/gif',
+		},
+	}
+
+	const newAsset = {
+		id: '1',
+		type: 'image',
+		props: {
+			src: 'https://www.youtube.com/watch?v=1',
+			name: 'image',
+			width: 100,
+			height: 100,
+			mimeType: 'image/gif',
+			fileSize: -1,
+		},
+	}
+
+	const { up, down } = getTestMigration(imageAssetVersions.AddFileSize)
+
+	test('up works as expected', () => {
+		expect(up(oldAsset)).toEqual(newAsset)
+	})
+	test('down works as expected', () => {
+		expect(down(newAsset)).toEqual(oldAsset)
+	})
+})
+
 const ShapeRecord = createRecordType('shape', {
 	validator: { validate: (record) => record as TLShape },
 	scope: 'document',

scripts/deploy.ts

@@ -24,6 +24,7 @@ const dotcom = path.relative(process.cwd(), path.resolve(__dirname, '../apps/dot
 const env = makeEnv([
 	'APP_ORIGIN',
 	'ASSET_UPLOAD',
+	'ASSET_BUCKET_ORIGIN',
 	'CLOUDFLARE_ACCOUNT_ID',
 	'CLOUDFLARE_API_TOKEN',
 	'DISCORD_DEPLOY_WEBHOOK_URL',