lod: make a hook available to rewrite image urls as needed

apps/dotcom/src/components/LocalEditor.tsx

@@ -13,6 +13,7 @@ import {
 	ExtrasGroup,
 	PreferencesGroup,
 	TLComponents,
+	TLHooks,
 	Tldraw,
 	TldrawUiMenuGroup,
 	TldrawUiMenuItem,
@@ -25,6 +26,7 @@ import { DebugMenuItems } from '../utils/migration/DebugMenuItems'
 import { LocalMigration } from '../utils/migration/LocalMigration'
 import { SCRATCH_PERSISTENCE_KEY } from '../utils/scratch-persistence-key'
 import { useSharing } from '../utils/sharing'
+import { useAssetHandler } from '../utils/useAssetHandler'
 import { OPEN_FILE_ACTION, SAVE_FILE_COPY_ACTION, useFileSystem } from '../utils/useFileSystem'
 import { useHandleUiEvents } from '../utils/useHandleUiEvent'
 import { LocalFileMenu } from './FileMenu'
@@ -85,6 +87,10 @@ const components: TLComponents = {
 	},
 }
 
+const hooks: TLHooks = {
+	useAssetHandler,
+}
+
 export function LocalEditor() {
 	const handleUiEvent = useHandleUiEvents()
 	const sharingUiOverrides = useSharing()
@@ -106,6 +112,7 @@ export function LocalEditor() {
 				overrides={[sharingUiOverrides, fileSystemUiOverrides]}
 				onUiEvent={handleUiEvent}
 				components={components}
+				hooks={hooks}
 				inferDarkMode
 			>
 				<LocalMigration />

apps/dotcom/src/components/MultiplayerEditor.tsx

@@ -14,6 +14,7 @@ import {
 	ExtrasGroup,
 	PreferencesGroup,
 	TLComponents,
+	TLHooks,
 	Tldraw,
 	TldrawUiMenuGroup,
 	TldrawUiMenuItem,
@@ -30,6 +31,7 @@ import { CursorChatMenuItem } from '../utils/context-menu/CursorChatMenuItem'
 import { createAssetFromFile } from '../utils/createAssetFromFile'
 import { createAssetFromUrl } from '../utils/createAssetFromUrl'
 import { useSharing } from '../utils/sharing'
+import { useAssetHandler } from '../utils/useAssetHandler'
 import { CURSOR_CHAT_ACTION, useCursorChat } from '../utils/useCursorChat'
 import { OPEN_FILE_ACTION, SAVE_FILE_COPY_ACTION, useFileSystem } from '../utils/useFileSystem'
 import { useHandleUiEvents } from '../utils/useHandleUiEvent'
@@ -103,6 +105,10 @@ const components: TLComponents = {
 	},
 }
 
+const hooks: TLHooks = {
+	useAssetHandler,
+}
+
 export function MultiplayerEditor({
 	roomOpenMode,
 	roomSlug,
@@ -158,6 +164,7 @@ export function MultiplayerEditor({
 				initialState={isReadonly ? 'hand' : 'select'}
 				onUiEvent={handleUiEvent}
 				components={components}
+				hooks={hooks}
 				autoFocus
 				inferDarkMode
 			>

apps/dotcom/src/utils/useAssetHandler.test.ts

@@ -0,0 +1,66 @@
+import { TLAsset } from 'tldraw'
+import { useAssetHandler } from './useAssetHandler'
+
+describe('useAssetHandler', () => {
+	const { handleAsset } = useAssetHandler()
+
+	it('should return an empty string if the asset is null', () => {
+		expect(handleAsset(null, { zoom: 1, dpr: 1, networkEffectiveType: '4g' })).toBe('')
+	})
+
+	it('should return an empty string if the asset is undefined', () => {
+		expect(handleAsset(undefined, { zoom: 1, dpr: 1, networkEffectiveType: '4g' })).toBe('')
+	})
+
+	it('should return an empty string if the asset has no src', () => {
+		const asset = { type: 'image', props: { w: 100 } }
+		expect(handleAsset(asset as TLAsset, { zoom: 1, dpr: 1, networkEffectiveType: '4g' })).toBe('')
+	})
+
+	it('should return the original src for video types', () => {
+		const asset = { type: 'video', props: { src: 'http://example.com/video.mp4' } }
+		expect(handleAsset(asset as TLAsset, { zoom: 1, dpr: 1, networkEffectiveType: '4g' })).toBe(
+			'http://example.com/video.mp4'
+		)
+	})
+
+	it('should return the original src if it does not start with http or https', () => {
+		const asset = { type: 'image', props: { src: 'data:somedata', w: 100 } }
+		expect(handleAsset(asset as TLAsset, { zoom: 1, dpr: 1, networkEffectiveType: '4g' })).toBe(
+			'data:somedata'
+		)
+	})
+
+	it("should return an empty string if the asset type is not 'image'", () => {
+		const asset = { type: 'document', props: { src: 'http://example.com/doc.pdf', w: 100 } }
+		expect(handleAsset(asset as TLAsset, { zoom: 1, dpr: 1, networkEffectiveType: '4g' })).toBe('')
+	})
+
+	it('should handle if network compensation is not available and zoom correctly', () => {
+		const asset = { type: 'image', props: { src: 'http://example.com/image.jpg', w: 100 } }
+		expect(handleAsset(asset as TLAsset, { zoom: 0.5, dpr: 2, networkEffectiveType: null })).toBe(
+			'https://localhost:8788/cdn-cgi/image/width=50,dpr=2,fit=scale-down,quality=92/http://example.com/image.jpg'
+		)
+	})
+
+	it('should handle network compensation and zoom correctly', () => {
+		const asset = { type: 'image', props: { src: 'http://example.com/image.jpg', w: 100 } }
+		expect(handleAsset(asset as TLAsset, { zoom: 0.5, dpr: 2, networkEffectiveType: '3g' })).toBe(
+			'https://localhost:8788/cdn-cgi/image/width=25,dpr=2,fit=scale-down,quality=92/http://example.com/image.jpg'
+		)
+	})
+
+	it('should round zoom to the nearest 0.25 and apply network compensation', () => {
+		const asset = { type: 'image', props: { src: 'https://example.com/image.jpg', w: 100 } }
+		expect(handleAsset(asset as TLAsset, { zoom: 0.33, dpr: 1, networkEffectiveType: '2g' })).toBe(
+			'https://localhost:8788/cdn-cgi/image/width=13,dpr=1,fit=scale-down,quality=92/https://example.com/image.jpg'
+		)
+	})
+
+	it('should set zoom to a minimum of 0.25 if zoom is below 0.25', () => {
+		const asset = { type: 'image', props: { src: 'https://example.com/image.jpg', w: 100 } }
+		expect(handleAsset(asset as TLAsset, { zoom: 0.1, dpr: 1, networkEffectiveType: '4g' })).toBe(
+			'https://localhost:8788/cdn-cgi/image/width=25,dpr=1,fit=scale-down,quality=92/https://example.com/image.jpg'
+		)
+	})
+})

apps/dotcom/src/utils/useAssetHandler.ts

@@ -0,0 +1,37 @@
+import { AssetContextProps, TLAsset } from 'tldraw'
+import { ASSET_UPLOADER_URL } from './config'
+
+export function useAssetHandler() {
+	const handleAsset = (asset: TLAsset | null | undefined, context: AssetContextProps) => {
+		if (!asset || !asset.props.src) return ''
+
+		// We don't deal with videos at the moment.
+		if (asset.type === 'video') return asset.props.src
+
+		// Assert it's an image to make TS happy.
+		if (asset.type !== 'image') return ''
+
+		// Don't try to transform data: URLs, yikes.
+		if (!asset.props.src.startsWith('http:') && !asset.props.src.startsWith('https:'))
+			return asset.props.src
+
+		// N.B. navigator.connection is only available in certain browsers (mainly Blink-based browsers)
+		// 4g is as high the 'effectiveType' goes and we can pick a lower effective image quality for slower connections.
+		const networkCompensation =
+			!context.networkEffectiveType || context.networkEffectiveType === '4g' ? 1 : 0.5
+
+		// We only look at the zoom level to the nearest 0.25
+		const zoomStepFunction = (zoom: number) => Math.floor(zoom * 4) / 4
+		const steppedZoom = Math.max(0.25, zoomStepFunction(context.zoom))
+
+		const width = Math.ceil(asset.props.w * steppedZoom * networkCompensation)
+
+		if (process.env.NODE_ENV === 'development') {
+			return asset.props.src
+		}
+
+		return `${ASSET_UPLOADER_URL}/cdn-cgi/image/width=${width},dpr=${context.dpr},fit=scale-down,quality=92/${asset.props.src}`
+	}
+
+	return { handleAsset }
+}

packages/editor/api-report.md

@@ -141,6 +141,13 @@ export class Arc2d extends Geometry2d {
 // @public
 export function areAnglesCompatible(a: number, b: number): boolean;
 
+// @public (undocumented)
+export type AssetContextProps = {
+    dpr: number;
+    networkEffectiveType: null | string;
+    zoom: number;
+};
+
 export { Atom }
 
 export { atom }
@@ -2233,6 +2240,7 @@ export interface TldrawEditorBaseProps {
     children?: ReactNode;
     className?: string;
     components?: TLEditorComponents;
+    hooks?: TLEditorHooks;
     inferDarkMode?: boolean;
     initialState?: string;
     onMount?: TLOnMountHandler;
@@ -2259,6 +2267,11 @@ export type TLEditorComponents = Partial<{
     [K in keyof BaseEditorComponents]: BaseEditorComponents[K] | null;
 } & ErrorComponents>;
 
+// @public (undocumented)
+export type TLEditorHooks = {
+    [K in keyof BaseEditorHooks]: BaseEditorHooks[K];
+};
+
 // @public (undocumented)
 export interface TLEditorOptions {
     bindingUtils: readonly TLBindingUtilConstructor<TLUnknownBinding>[];
@@ -2869,6 +2882,9 @@ export function useEditorComponents(): Partial<{
     ZoomBrush: ComponentType<TLBrushProps> | null;
 } & ErrorComponents> & ErrorComponents;
 
+// @public (undocumented)
+export function useEditorHooks(): TLEditorHooks;
+
 // @internal
 export function useEvent<Args extends Array<unknown>, Result>(handler: (...args: Args) => Result): (...args: Args) => Result;
 

packages/editor/src/index.ts

@@ -243,6 +243,8 @@ export { getCursor } from './lib/hooks/useCursor'
 export { EditorContext, useEditor } from './lib/hooks/useEditor'
 export { useEditorComponents } from './lib/hooks/useEditorComponents'
 export type { TLEditorComponents } from './lib/hooks/useEditorComponents'
+export { useEditorHooks } from './lib/hooks/useEditorHooks'
+export type { AssetContextProps, TLEditorHooks } from './lib/hooks/useEditorHooks'
 export { useEvent } from './lib/hooks/useEvent'
 export { useShallowArrayIdentity, useShallowObjectIdentity } from './lib/hooks/useIdentity'
 export { useIsCropping } from './lib/hooks/useIsCropping'

packages/editor/src/lib/TldrawEditor.tsx

@@ -29,6 +29,7 @@ import {
 	TLEditorComponents,
 	useEditorComponents,
 } from './hooks/useEditorComponents'
+import { EditorHooksProvider, TLEditorHooks } from './hooks/useEditorHooks'
 import { useEvent } from './hooks/useEvent'
 import { useFocusEvents } from './hooks/useFocusEvents'
 import { useForceUpdate } from './hooks/useForceUpdate'
@@ -97,6 +98,12 @@ export interface TldrawEditorBaseProps {
 	 */
 	components?: TLEditorComponents
 
+	/**
+	 * Hooks into various logical entrypoints in the codebase.
+	 * For example, we allow hooking into asset url handling.
+	 */
+	hooks?: TLEditorHooks
+
 	/**
 	 * Called when the editor has mounted.
 	 */
@@ -154,6 +161,7 @@ const EMPTY_TOOLS_ARRAY = [] as const
 export const TldrawEditor = memo(function TldrawEditor({
 	store,
 	components,
+	hooks,
 	className,
 	user: _user,
 	...rest
@@ -190,18 +198,20 @@ export const TldrawEditor = memo(function TldrawEditor({
 				{container && (
 					<ContainerProvider container={container}>
 						<EditorComponentsProvider overrides={components}>
-							{store ? (
-								store instanceof Store ? (
-									// Store is ready to go, whether externally synced or not
-									<TldrawEditorWithReadyStore {...withDefaults} store={store} user={user} />
+							<EditorHooksProvider overrides={hooks}>
+								{store ? (
+									store instanceof Store ? (
+										// Store is ready to go, whether externally synced or not
+										<TldrawEditorWithReadyStore {...withDefaults} store={store} user={user} />
+									) : (
+										// Store is a synced store, so handle syncing stages internally
+										<TldrawEditorWithLoadingStore {...withDefaults} store={store} user={user} />
+									)
 								) : (
-									// Store is a synced store, so handle syncing stages internally
-									<TldrawEditorWithLoadingStore {...withDefaults} store={store} user={user} />
-								)
-							) : (
-								// We have no store (it's undefined) so create one and possibly sync it
-								<TldrawEditorWithOwnStore {...withDefaults} store={store} user={user} />
-							)}
+									// We have no store (it's undefined) so create one and possibly sync it
+									<TldrawEditorWithOwnStore {...withDefaults} store={store} user={user} />
+								)}
+							</EditorHooksProvider>
 						</EditorComponentsProvider>
 					</ContainerProvider>
 				)}

packages/editor/src/lib/hooks/useEditorHooks.tsx

@@ -0,0 +1,58 @@
+import { TLAsset } from '@tldraw/tlschema'
+import { createContext, useContext, useMemo } from 'react'
+import { useShallowObjectIdentity } from './useIdentity'
+
+/** @public */
+export type AssetContextProps = {
+	zoom: number
+	dpr: number
+	networkEffectiveType: string | null
+}
+
+/** @public */
+export type TLAssetHandlerHook = () => {
+	handleAsset: (asset: TLAsset | null | undefined, context: AssetContextProps) => string
+}
+
+export interface BaseEditorHooks {
+	useAssetHandler: TLAssetHandlerHook
+}
+
+/** @public */
+export type TLEditorHooks = {
+	[K in keyof BaseEditorHooks]: BaseEditorHooks[K]
+}
+
+const EditorHooksContext = createContext({} as TLEditorHooks)
+
+type HooksContextProviderProps = {
+	overrides?: TLEditorHooks
+	children: any
+}
+
+// Default handler is just a pass-through function.
+const DefaultAssetHandlerHook = () => ({
+	handleAsset: (asset: TLAsset | null | undefined) => asset?.props.src || '',
+})
+
+export function EditorHooksProvider({ overrides, children }: HooksContextProviderProps) {
+	const _overrides = useShallowObjectIdentity(overrides || {})
+	return (
+		<EditorHooksContext.Provider
+			value={useMemo(
+				() => ({
+					useAssetHandler: DefaultAssetHandlerHook,
+					..._overrides,
+				}),
+				[_overrides]
+			)}
+		>
+			{children}
+		</EditorHooksContext.Provider>
+	)
+}
+
+/** @public */
+export function useEditorHooks() {
+	return useContext(EditorHooksContext)
+}

packages/tldraw/api-report.md

@@ -76,6 +76,7 @@ import { TldrawEditorBaseProps } from '@tldraw/editor';
 import { TLDrawShape } from '@tldraw/editor';
 import { TLDrawShapeSegment } from '@tldraw/editor';
 import { TLEditorComponents } from '@tldraw/editor';
+import { TLEditorHooks } from '@tldraw/editor';
 import { TLEmbedShape } from '@tldraw/editor';
 import { TLEnterEventHandler } from '@tldraw/editor';
 import { TLEventHandlers } from '@tldraw/editor';
@@ -1799,6 +1800,9 @@ export const TldrawUiSlider: NamedExoticComponent<TLUiSliderProps>;
 // @public (undocumented)
 export type TLExportType = 'jpeg' | 'json' | 'png' | 'svg' | 'webp';
 
+// @public (undocumented)
+export type TLHooks = Expand<TLEditorHooks>;
+
 // @public (undocumented)
 export interface TLUiActionItem<TransationKey extends string = string, IconType extends string = string> {
     // (undocumented)

packages/tldraw/src/index.ts

@@ -292,7 +292,7 @@ export {
 	FeatureFlags,
 } from './lib/ui/components/DebugMenu/DefaultDebugMenuContent'
 
-export { type TLComponents } from './lib/Tldraw'
+export { type TLComponents, type TLHooks } from './lib/Tldraw'
 
 /* ------------------- Primitives ------------------- */
 

packages/tldraw/src/lib/Tldraw.tsx

@@ -8,6 +8,7 @@ import {
 	MigrationSequence,
 	StoreSnapshot,
 	TLEditorComponents,
+	TLEditorHooks,
 	TLOnMountHandler,
 	TLRecord,
 	TLStore,
@@ -44,6 +45,9 @@ import { useDefaultEditorAssetsWithOverrides } from './utils/static-assets/asset
 /**@public */
 export type TLComponents = Expand<TLEditorComponents & TLUiComponents>
 
+/**@public */
+export type TLHooks = Expand<TLEditorHooks>
+
 /** @public */
 export type TldrawProps = Expand<
 	// combine components from base editor and ui

packages/tldraw/src/lib/shapes/image/ImageShapeUtil.tsx

@@ -8,12 +8,15 @@ import {
 	TLOnDoubleClickHandler,
 	TLShapePartial,
 	Vec,
+	debounce,
 	imageShapeMigrations,
 	imageShapeProps,
 	structuredClone,
 	toDomPrecision,
+	useEditorHooks,
+	useValue,
 } from '@tldraw/editor'
-import { useEffect, useState } from 'react'
+import { useEffect, useRef, useState } from 'react'
 import { BrokenAssetIcon } from '../shared/BrokenAssetIcon'
 import { HyperlinkButton } from '../shared/HyperlinkButton'
 import { usePrefersReducedMotion } from '../shared/usePrefersReducedMotion'
@@ -59,16 +62,51 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 		const isCropping = this.editor.getCroppingShapeId() === shape.id
 		const prefersReducedMotion = usePrefersReducedMotion()
 		const [staticFrameSrc, setStaticFrameSrc] = useState('')
+		const [loadedSrc, setLoadedSrc] = useState('')
+		const { useAssetHandler } = useEditorHooks()
+		const { handleAsset } = useAssetHandler()
 
 		const asset = shape.props.assetId ? this.editor.getAsset(shape.props.assetId) : undefined
 
 		const isSelected = shape.id === this.editor.getOnlySelectedShapeId()
 
+		// We debounce the zoom level to reduce the number of times we fetch a new image and,
+		// more importantly, to not cause zooming in and out to feel janky.
+		const [debouncedZoom, setDebouncedZoom] = useState(this.editor.getZoomLevel())
+		const zoomUpdater = useRef(debounce((zoom: number) => setDebouncedZoom(zoom), 500))
+		useValue('zoom level', () => zoomUpdater.current(this.editor.getZoomLevel()), [this.editor])
+		const networkEffectiveType: string | null =
+			'connection' in navigator ? (navigator as any).connection.effectiveType : null
+		const dpr = window.devicePixelRatio
+		const src = handleAsset(asset, { zoom: debouncedZoom, dpr, networkEffectiveType })
+
 		useEffect(() => {
-			if (asset?.props.src && this.isAnimated(shape)) {
+			// If an image is not animated (that's handled below), then we preload the image
+			// because we might have different source urls for different zoom levels.
+			// Preloading the image ensures that the browser caches the image and doesn't
+			// cause visual flickering when the image is loaded.
+			if (src && !this.isAnimated(shape)) {
 				let cancelled = false
-				const url = asset.props.src
-				if (!url) return
+				if (!src) return
+
+				const image = new Image()
+				image.onload = () => {
+					if (cancelled) return
+					setLoadedSrc(src)
+				}
+				image.crossOrigin = 'anonymous'
+				image.src = src
+
+				return () => {
+					cancelled = true
+				}
+			}
+		}, [src, shape])
+
+		useEffect(() => {
+			if (src && this.isAnimated(shape)) {
+				let cancelled = false
+				if (!src) return
 
 				const image = new Image()
 				image.onload = () => {
@@ -85,13 +123,13 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 					setStaticFrameSrc(canvas.toDataURL())
 				}
 				image.crossOrigin = 'anonymous'
-				image.src = url
+				image.src = src
 
 				return () => {
 					cancelled = true
 				}
 			}
-		}, [prefersReducedMotion, asset?.props, shape])
+		}, [prefersReducedMotion, src, shape])
 
 		if (asset?.type === 'bookmark') {
 			throw Error("Bookmark assets can't be rendered as images")
@@ -108,7 +146,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 
 		const containerStyle = getCroppedContainerStyle(shape)
 
-		if (!asset?.props.src) {
+		if (!src) {
 			return (
 				<HTMLContainer
 					id={shape.id}
@@ -141,7 +179,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 							style={{
 								opacity: 0.1,
 								backgroundImage: `url(${
-									!shape.props.playing || reduceMotion ? staticFrameSrc : asset.props.src
+									!shape.props.playing || reduceMotion ? staticFrameSrc : loadedSrc
 								})`,
 							}}
 							draggable={false}
@@ -157,7 +195,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 							className="tl-image"
 							style={{
 								backgroundImage: `url(${
-									!shape.props.playing || reduceMotion ? staticFrameSrc : asset.props.src
+									!shape.props.playing || reduceMotion ? staticFrameSrc : loadedSrc
 								})`,
 							}}
 							draggable={false}

packages/tldraw/src/lib/shapes/video/VideoShapeUtil.tsx

@@ -3,8 +3,11 @@ import {
 	BaseBoxShapeUtil,
 	HTMLContainer,
 	TLVideoShape,
+	debounce,
 	toDomPrecision,
+	useEditorHooks,
 	useIsEditing,
+	useValue,
 	videoShapeMigrations,
 	videoShapeProps,
 } from '@tldraw/editor'
@@ -40,6 +43,8 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
 		const { time, playing } = shape.props
 		const isEditing = useIsEditing(shape.id)
 		const prefersReducedMotion = usePrefersReducedMotion()
+		const { useAssetHandler } = useEditorHooks()
+		const { handleAsset } = useAssetHandler()
 
 		const rVideo = useRef<HTMLVideoElement>(null!)
 
@@ -145,6 +150,19 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
 			}
 		}, [rVideo, prefersReducedMotion])
 
+		// We debounce the zoom level to reduce the number of times we fetch a new image and,
+		// more importantly, to not cause zooming in and out to feel janky.
+		const [debouncedZoom, setDebouncedZoom] = useState(this.editor.getZoomLevel())
+		const zoomUpdater = useRef(debounce((zoom: number) => setDebouncedZoom(zoom), 500))
+		useValue('zoom level', () => zoomUpdater.current(editor.getZoomLevel()), [editor])
+		const networkEffectiveType: string | null =
+			'connection' in navigator ? (navigator as any).connection.effectiveType : null
+		const dpr = window.devicePixelRatio
+
+		// N.B. on dotcom, we don't currently actually change the quality of the video at the moment.
+		// But theoretically, someone could take advantage of this capability for videos.
+		const src = handleAsset(asset, { zoom: debouncedZoom, dpr, networkEffectiveType })
+
 		return (
 			<>
 				<HTMLContainer
@@ -157,7 +175,7 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
 				>
 					<div className="tl-counter-scaled">
 						<div className="tl-video-container">
-							{asset?.props.src ? (
+							{src ? (
 								<video
 									ref={rVideo}
 									style={isEditing ? { pointerEvents: 'all' } : undefined}
@@ -178,7 +196,7 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
 									onLoadedData={handleLoadedData}
 									hidden={!isLoaded}
 								>
-									<source src={asset.props.src} />
+									<source src={src} />
 								</video>
 							) : (
 								<BrokenAssetIcon />