diff --git a/apps/dotcom/src/components/LocalEditor.tsx b/apps/dotcom/src/components/LocalEditor.tsx index fc42eb00a..b1c1d5c1b 100644 --- a/apps/dotcom/src/components/LocalEditor.tsx +++ b/apps/dotcom/src/components/LocalEditor.tsx @@ -13,6 +13,7 @@ import { ExtrasGroup, PreferencesGroup, TLComponents, + TLHooks, Tldraw, TldrawUiMenuGroup, TldrawUiMenuItem, @@ -25,6 +26,7 @@ import { DebugMenuItems } from '../utils/migration/DebugMenuItems' import { LocalMigration } from '../utils/migration/LocalMigration' import { SCRATCH_PERSISTENCE_KEY } from '../utils/scratch-persistence-key' import { useSharing } from '../utils/sharing' +import { useAssetHandler } from '../utils/useAssetHandler' import { OPEN_FILE_ACTION, SAVE_FILE_COPY_ACTION, useFileSystem } from '../utils/useFileSystem' import { useHandleUiEvents } from '../utils/useHandleUiEvent' import { LocalFileMenu } from './FileMenu' @@ -85,6 +87,10 @@ const components: TLComponents = { }, } +const hooks: TLHooks = { + useAssetHandler, +} + export function LocalEditor() { const handleUiEvent = useHandleUiEvents() const sharingUiOverrides = useSharing() @@ -106,6 +112,7 @@ export function LocalEditor() { overrides={[sharingUiOverrides, fileSystemUiOverrides]} onUiEvent={handleUiEvent} components={components} + hooks={hooks} inferDarkMode > diff --git a/apps/dotcom/src/components/MultiplayerEditor.tsx b/apps/dotcom/src/components/MultiplayerEditor.tsx index 10f53df4e..fe5f46822 100644 --- a/apps/dotcom/src/components/MultiplayerEditor.tsx +++ b/apps/dotcom/src/components/MultiplayerEditor.tsx @@ -14,6 +14,7 @@ import { ExtrasGroup, PreferencesGroup, TLComponents, + TLHooks, Tldraw, TldrawUiMenuGroup, TldrawUiMenuItem, @@ -30,6 +31,7 @@ import { CursorChatMenuItem } from '../utils/context-menu/CursorChatMenuItem' import { createAssetFromFile } from '../utils/createAssetFromFile' import { createAssetFromUrl } from '../utils/createAssetFromUrl' import { useSharing } from '../utils/sharing' +import { useAssetHandler } from '../utils/useAssetHandler' import { CURSOR_CHAT_ACTION, useCursorChat } from '../utils/useCursorChat' import { OPEN_FILE_ACTION, SAVE_FILE_COPY_ACTION, useFileSystem } from '../utils/useFileSystem' import { useHandleUiEvents } from '../utils/useHandleUiEvent' @@ -103,6 +105,10 @@ const components: TLComponents = { }, } +const hooks: TLHooks = { + useAssetHandler, +} + export function MultiplayerEditor({ roomOpenMode, roomSlug, @@ -158,6 +164,7 @@ export function MultiplayerEditor({ initialState={isReadonly ? 'hand' : 'select'} onUiEvent={handleUiEvent} components={components} + hooks={hooks} autoFocus inferDarkMode > diff --git a/apps/dotcom/src/utils/useAssetHandler.test.ts b/apps/dotcom/src/utils/useAssetHandler.test.ts new file mode 100644 index 000000000..ad5f45c8c --- /dev/null +++ b/apps/dotcom/src/utils/useAssetHandler.test.ts @@ -0,0 +1,66 @@ +import { TLAsset } from 'tldraw' +import { useAssetHandler } from './useAssetHandler' + +describe('useAssetHandler', () => { + const { handleAsset } = useAssetHandler() + + it('should return an empty string if the asset is null', () => { + expect(handleAsset(null, { zoom: 1, dpr: 1, networkEffectiveType: '4g' })).toBe('') + }) + + it('should return an empty string if the asset is undefined', () => { + expect(handleAsset(undefined, { zoom: 1, dpr: 1, networkEffectiveType: '4g' })).toBe('') + }) + + it('should return an empty string if the asset has no src', () => { + const asset = { type: 'image', props: { w: 100 } } + expect(handleAsset(asset as TLAsset, { zoom: 1, dpr: 1, networkEffectiveType: '4g' })).toBe('') + }) + + it('should return the original src for video types', () => { + const asset = { type: 'video', props: { src: 'http://example.com/video.mp4' } } + expect(handleAsset(asset as TLAsset, { zoom: 1, dpr: 1, networkEffectiveType: '4g' })).toBe( + 'http://example.com/video.mp4' + ) + }) + + it('should return the original src if it does not start with http or https', () => { + const asset = { type: 'image', props: { src: 'data:somedata', w: 100 } } + expect(handleAsset(asset as TLAsset, { zoom: 1, dpr: 1, networkEffectiveType: '4g' })).toBe( + 'data:somedata' + ) + }) + + it("should return an empty string if the asset type is not 'image'", () => { + const asset = { type: 'document', props: { src: 'http://example.com/doc.pdf', w: 100 } } + expect(handleAsset(asset as TLAsset, { zoom: 1, dpr: 1, networkEffectiveType: '4g' })).toBe('') + }) + + it('should handle if network compensation is not available and zoom correctly', () => { + const asset = { type: 'image', props: { src: 'http://example.com/image.jpg', w: 100 } } + expect(handleAsset(asset as TLAsset, { zoom: 0.5, dpr: 2, networkEffectiveType: null })).toBe( + 'https://localhost:8788/cdn-cgi/image/width=50,dpr=2,fit=scale-down,quality=92/http://example.com/image.jpg' + ) + }) + + it('should handle network compensation and zoom correctly', () => { + const asset = { type: 'image', props: { src: 'http://example.com/image.jpg', w: 100 } } + expect(handleAsset(asset as TLAsset, { zoom: 0.5, dpr: 2, networkEffectiveType: '3g' })).toBe( + 'https://localhost:8788/cdn-cgi/image/width=25,dpr=2,fit=scale-down,quality=92/http://example.com/image.jpg' + ) + }) + + it('should round zoom to the nearest 0.25 and apply network compensation', () => { + const asset = { type: 'image', props: { src: 'https://example.com/image.jpg', w: 100 } } + expect(handleAsset(asset as TLAsset, { zoom: 0.33, dpr: 1, networkEffectiveType: '2g' })).toBe( + 'https://localhost:8788/cdn-cgi/image/width=13,dpr=1,fit=scale-down,quality=92/https://example.com/image.jpg' + ) + }) + + it('should set zoom to a minimum of 0.25 if zoom is below 0.25', () => { + const asset = { type: 'image', props: { src: 'https://example.com/image.jpg', w: 100 } } + expect(handleAsset(asset as TLAsset, { zoom: 0.1, dpr: 1, networkEffectiveType: '4g' })).toBe( + 'https://localhost:8788/cdn-cgi/image/width=25,dpr=1,fit=scale-down,quality=92/https://example.com/image.jpg' + ) + }) +}) diff --git a/apps/dotcom/src/utils/useAssetHandler.ts b/apps/dotcom/src/utils/useAssetHandler.ts new file mode 100644 index 000000000..99010eca4 --- /dev/null +++ b/apps/dotcom/src/utils/useAssetHandler.ts @@ -0,0 +1,37 @@ +import { AssetContextProps, TLAsset } from 'tldraw' +import { ASSET_UPLOADER_URL } from './config' + +export function useAssetHandler() { + const handleAsset = (asset: TLAsset | null | undefined, context: AssetContextProps) => { + if (!asset || !asset.props.src) return '' + + // We don't deal with videos at the moment. + if (asset.type === 'video') return asset.props.src + + // Assert it's an image to make TS happy. + if (asset.type !== 'image') return '' + + // Don't try to transform data: URLs, yikes. + if (!asset.props.src.startsWith('http:') && !asset.props.src.startsWith('https:')) + return asset.props.src + + // N.B. navigator.connection is only available in certain browsers (mainly Blink-based browsers) + // 4g is as high the 'effectiveType' goes and we can pick a lower effective image quality for slower connections. + const networkCompensation = + !context.networkEffectiveType || context.networkEffectiveType === '4g' ? 1 : 0.5 + + // We only look at the zoom level to the nearest 0.25 + const zoomStepFunction = (zoom: number) => Math.floor(zoom * 4) / 4 + const steppedZoom = Math.max(0.25, zoomStepFunction(context.zoom)) + + const width = Math.ceil(asset.props.w * steppedZoom * networkCompensation) + + if (process.env.NODE_ENV === 'development') { + return asset.props.src + } + + return `${ASSET_UPLOADER_URL}/cdn-cgi/image/width=${width},dpr=${context.dpr},fit=scale-down,quality=92/${asset.props.src}` + } + + return { handleAsset } +} diff --git a/packages/editor/api-report.md b/packages/editor/api-report.md index 28ec3a425..a0a6be8b4 100644 --- a/packages/editor/api-report.md +++ b/packages/editor/api-report.md @@ -141,6 +141,13 @@ export class Arc2d extends Geometry2d { // @public export function areAnglesCompatible(a: number, b: number): boolean; +// @public (undocumented) +export type AssetContextProps = { + dpr: number; + networkEffectiveType: null | string; + zoom: number; +}; + export { Atom } export { atom } @@ -2233,6 +2240,7 @@ export interface TldrawEditorBaseProps { children?: ReactNode; className?: string; components?: TLEditorComponents; + hooks?: TLEditorHooks; inferDarkMode?: boolean; initialState?: string; onMount?: TLOnMountHandler; @@ -2259,6 +2267,11 @@ export type TLEditorComponents = Partial<{ [K in keyof BaseEditorComponents]: BaseEditorComponents[K] | null; } & ErrorComponents>; +// @public (undocumented) +export type TLEditorHooks = { + [K in keyof BaseEditorHooks]: BaseEditorHooks[K]; +}; + // @public (undocumented) export interface TLEditorOptions { bindingUtils: readonly TLBindingUtilConstructor[]; @@ -2869,6 +2882,9 @@ export function useEditorComponents(): Partial<{ ZoomBrush: ComponentType | null; } & ErrorComponents> & ErrorComponents; +// @public (undocumented) +export function useEditorHooks(): TLEditorHooks; + // @internal export function useEvent, Result>(handler: (...args: Args) => Result): (...args: Args) => Result; diff --git a/packages/editor/src/index.ts b/packages/editor/src/index.ts index 4ef5a26ea..23b6da7b3 100644 --- a/packages/editor/src/index.ts +++ b/packages/editor/src/index.ts @@ -243,6 +243,8 @@ export { getCursor } from './lib/hooks/useCursor' export { EditorContext, useEditor } from './lib/hooks/useEditor' export { useEditorComponents } from './lib/hooks/useEditorComponents' export type { TLEditorComponents } from './lib/hooks/useEditorComponents' +export { useEditorHooks } from './lib/hooks/useEditorHooks' +export type { AssetContextProps, TLEditorHooks } from './lib/hooks/useEditorHooks' export { useEvent } from './lib/hooks/useEvent' export { useShallowArrayIdentity, useShallowObjectIdentity } from './lib/hooks/useIdentity' export { useIsCropping } from './lib/hooks/useIsCropping' diff --git a/packages/editor/src/lib/TldrawEditor.tsx b/packages/editor/src/lib/TldrawEditor.tsx index 565a73607..7725fbe24 100644 --- a/packages/editor/src/lib/TldrawEditor.tsx +++ b/packages/editor/src/lib/TldrawEditor.tsx @@ -29,6 +29,7 @@ import { TLEditorComponents, useEditorComponents, } from './hooks/useEditorComponents' +import { EditorHooksProvider, TLEditorHooks } from './hooks/useEditorHooks' import { useEvent } from './hooks/useEvent' import { useFocusEvents } from './hooks/useFocusEvents' import { useForceUpdate } from './hooks/useForceUpdate' @@ -97,6 +98,12 @@ export interface TldrawEditorBaseProps { */ components?: TLEditorComponents + /** + * Hooks into various logical entrypoints in the codebase. + * For example, we allow hooking into asset url handling. + */ + hooks?: TLEditorHooks + /** * Called when the editor has mounted. */ @@ -154,6 +161,7 @@ const EMPTY_TOOLS_ARRAY = [] as const export const TldrawEditor = memo(function TldrawEditor({ store, components, + hooks, className, user: _user, ...rest @@ -190,18 +198,20 @@ export const TldrawEditor = memo(function TldrawEditor({ {container && ( - {store ? ( - store instanceof Store ? ( - // Store is ready to go, whether externally synced or not - + + {store ? ( + store instanceof Store ? ( + // Store is ready to go, whether externally synced or not + + ) : ( + // Store is a synced store, so handle syncing stages internally + + ) ) : ( - // Store is a synced store, so handle syncing stages internally - - ) - ) : ( - // We have no store (it's undefined) so create one and possibly sync it - - )} + // We have no store (it's undefined) so create one and possibly sync it + + )} + )} diff --git a/packages/editor/src/lib/hooks/useEditorHooks.tsx b/packages/editor/src/lib/hooks/useEditorHooks.tsx new file mode 100644 index 000000000..425f20419 --- /dev/null +++ b/packages/editor/src/lib/hooks/useEditorHooks.tsx @@ -0,0 +1,58 @@ +import { TLAsset } from '@tldraw/tlschema' +import { createContext, useContext, useMemo } from 'react' +import { useShallowObjectIdentity } from './useIdentity' + +/** @public */ +export type AssetContextProps = { + zoom: number + dpr: number + networkEffectiveType: string | null +} + +/** @public */ +export type TLAssetHandlerHook = () => { + handleAsset: (asset: TLAsset | null | undefined, context: AssetContextProps) => string +} + +export interface BaseEditorHooks { + useAssetHandler: TLAssetHandlerHook +} + +/** @public */ +export type TLEditorHooks = { + [K in keyof BaseEditorHooks]: BaseEditorHooks[K] +} + +const EditorHooksContext = createContext({} as TLEditorHooks) + +type HooksContextProviderProps = { + overrides?: TLEditorHooks + children: any +} + +// Default handler is just a pass-through function. +const DefaultAssetHandlerHook = () => ({ + handleAsset: (asset: TLAsset | null | undefined) => asset?.props.src || '', +}) + +export function EditorHooksProvider({ overrides, children }: HooksContextProviderProps) { + const _overrides = useShallowObjectIdentity(overrides || {}) + return ( + ({ + useAssetHandler: DefaultAssetHandlerHook, + ..._overrides, + }), + [_overrides] + )} + > + {children} + + ) +} + +/** @public */ +export function useEditorHooks() { + return useContext(EditorHooksContext) +} diff --git a/packages/tldraw/api-report.md b/packages/tldraw/api-report.md index e5a364429..12895f75c 100644 --- a/packages/tldraw/api-report.md +++ b/packages/tldraw/api-report.md @@ -76,6 +76,7 @@ import { TldrawEditorBaseProps } from '@tldraw/editor'; import { TLDrawShape } from '@tldraw/editor'; import { TLDrawShapeSegment } from '@tldraw/editor'; import { TLEditorComponents } from '@tldraw/editor'; +import { TLEditorHooks } from '@tldraw/editor'; import { TLEmbedShape } from '@tldraw/editor'; import { TLEnterEventHandler } from '@tldraw/editor'; import { TLEventHandlers } from '@tldraw/editor'; @@ -1799,6 +1800,9 @@ export const TldrawUiSlider: NamedExoticComponent; // @public (undocumented) export type TLExportType = 'jpeg' | 'json' | 'png' | 'svg' | 'webp'; +// @public (undocumented) +export type TLHooks = Expand; + // @public (undocumented) export interface TLUiActionItem { // (undocumented) diff --git a/packages/tldraw/src/index.ts b/packages/tldraw/src/index.ts index 72524d59f..cd4481582 100644 --- a/packages/tldraw/src/index.ts +++ b/packages/tldraw/src/index.ts @@ -292,7 +292,7 @@ export { FeatureFlags, } from './lib/ui/components/DebugMenu/DefaultDebugMenuContent' -export { type TLComponents } from './lib/Tldraw' +export { type TLComponents, type TLHooks } from './lib/Tldraw' /* ------------------- Primitives ------------------- */ diff --git a/packages/tldraw/src/lib/Tldraw.tsx b/packages/tldraw/src/lib/Tldraw.tsx index 85b0a3ff9..03c134695 100644 --- a/packages/tldraw/src/lib/Tldraw.tsx +++ b/packages/tldraw/src/lib/Tldraw.tsx @@ -8,6 +8,7 @@ import { MigrationSequence, StoreSnapshot, TLEditorComponents, + TLEditorHooks, TLOnMountHandler, TLRecord, TLStore, @@ -44,6 +45,9 @@ import { useDefaultEditorAssetsWithOverrides } from './utils/static-assets/asset /**@public */ export type TLComponents = Expand +/**@public */ +export type TLHooks = Expand + /** @public */ export type TldrawProps = Expand< // combine components from base editor and ui diff --git a/packages/tldraw/src/lib/shapes/image/ImageShapeUtil.tsx b/packages/tldraw/src/lib/shapes/image/ImageShapeUtil.tsx index a69c9a660..79e3035de 100644 --- a/packages/tldraw/src/lib/shapes/image/ImageShapeUtil.tsx +++ b/packages/tldraw/src/lib/shapes/image/ImageShapeUtil.tsx @@ -8,12 +8,15 @@ import { TLOnDoubleClickHandler, TLShapePartial, Vec, + debounce, imageShapeMigrations, imageShapeProps, structuredClone, toDomPrecision, + useEditorHooks, + useValue, } from '@tldraw/editor' -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { BrokenAssetIcon } from '../shared/BrokenAssetIcon' import { HyperlinkButton } from '../shared/HyperlinkButton' import { usePrefersReducedMotion } from '../shared/usePrefersReducedMotion' @@ -59,16 +62,51 @@ export class ImageShapeUtil extends BaseBoxShapeUtil { const isCropping = this.editor.getCroppingShapeId() === shape.id const prefersReducedMotion = usePrefersReducedMotion() const [staticFrameSrc, setStaticFrameSrc] = useState('') + const [loadedSrc, setLoadedSrc] = useState('') + const { useAssetHandler } = useEditorHooks() + const { handleAsset } = useAssetHandler() const asset = shape.props.assetId ? this.editor.getAsset(shape.props.assetId) : undefined const isSelected = shape.id === this.editor.getOnlySelectedShapeId() + // We debounce the zoom level to reduce the number of times we fetch a new image and, + // more importantly, to not cause zooming in and out to feel janky. + const [debouncedZoom, setDebouncedZoom] = useState(this.editor.getZoomLevel()) + const zoomUpdater = useRef(debounce((zoom: number) => setDebouncedZoom(zoom), 500)) + useValue('zoom level', () => zoomUpdater.current(this.editor.getZoomLevel()), [this.editor]) + const networkEffectiveType: string | null = + 'connection' in navigator ? (navigator as any).connection.effectiveType : null + const dpr = window.devicePixelRatio + const src = handleAsset(asset, { zoom: debouncedZoom, dpr, networkEffectiveType }) + useEffect(() => { - if (asset?.props.src && this.isAnimated(shape)) { + // If an image is not animated (that's handled below), then we preload the image + // because we might have different source urls for different zoom levels. + // Preloading the image ensures that the browser caches the image and doesn't + // cause visual flickering when the image is loaded. + if (src && !this.isAnimated(shape)) { let cancelled = false - const url = asset.props.src - if (!url) return + if (!src) return + + const image = new Image() + image.onload = () => { + if (cancelled) return + setLoadedSrc(src) + } + image.crossOrigin = 'anonymous' + image.src = src + + return () => { + cancelled = true + } + } + }, [src, shape]) + + useEffect(() => { + if (src && this.isAnimated(shape)) { + let cancelled = false + if (!src) return const image = new Image() image.onload = () => { @@ -85,13 +123,13 @@ export class ImageShapeUtil extends BaseBoxShapeUtil { setStaticFrameSrc(canvas.toDataURL()) } image.crossOrigin = 'anonymous' - image.src = url + image.src = src return () => { cancelled = true } } - }, [prefersReducedMotion, asset?.props, shape]) + }, [prefersReducedMotion, src, shape]) if (asset?.type === 'bookmark') { throw Error("Bookmark assets can't be rendered as images") @@ -108,7 +146,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil { const containerStyle = getCroppedContainerStyle(shape) - if (!asset?.props.src) { + if (!src) { return ( { style={{ opacity: 0.1, backgroundImage: `url(${ - !shape.props.playing || reduceMotion ? staticFrameSrc : asset.props.src + !shape.props.playing || reduceMotion ? staticFrameSrc : loadedSrc })`, }} draggable={false} @@ -157,7 +195,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil { className="tl-image" style={{ backgroundImage: `url(${ - !shape.props.playing || reduceMotion ? staticFrameSrc : asset.props.src + !shape.props.playing || reduceMotion ? staticFrameSrc : loadedSrc })`, }} draggable={false} diff --git a/packages/tldraw/src/lib/shapes/video/VideoShapeUtil.tsx b/packages/tldraw/src/lib/shapes/video/VideoShapeUtil.tsx index ac88bb899..fcd874f00 100644 --- a/packages/tldraw/src/lib/shapes/video/VideoShapeUtil.tsx +++ b/packages/tldraw/src/lib/shapes/video/VideoShapeUtil.tsx @@ -3,8 +3,11 @@ import { BaseBoxShapeUtil, HTMLContainer, TLVideoShape, + debounce, toDomPrecision, + useEditorHooks, useIsEditing, + useValue, videoShapeMigrations, videoShapeProps, } from '@tldraw/editor' @@ -40,6 +43,8 @@ export class VideoShapeUtil extends BaseBoxShapeUtil { const { time, playing } = shape.props const isEditing = useIsEditing(shape.id) const prefersReducedMotion = usePrefersReducedMotion() + const { useAssetHandler } = useEditorHooks() + const { handleAsset } = useAssetHandler() const rVideo = useRef(null!) @@ -145,6 +150,19 @@ export class VideoShapeUtil extends BaseBoxShapeUtil { } }, [rVideo, prefersReducedMotion]) + // We debounce the zoom level to reduce the number of times we fetch a new image and, + // more importantly, to not cause zooming in and out to feel janky. + const [debouncedZoom, setDebouncedZoom] = useState(this.editor.getZoomLevel()) + const zoomUpdater = useRef(debounce((zoom: number) => setDebouncedZoom(zoom), 500)) + useValue('zoom level', () => zoomUpdater.current(editor.getZoomLevel()), [editor]) + const networkEffectiveType: string | null = + 'connection' in navigator ? (navigator as any).connection.effectiveType : null + const dpr = window.devicePixelRatio + + // N.B. on dotcom, we don't currently actually change the quality of the video at the moment. + // But theoretically, someone could take advantage of this capability for videos. + const src = handleAsset(asset, { zoom: debouncedZoom, dpr, networkEffectiveType }) + return ( <> { >
- {asset?.props.src ? ( + {src ? ( ) : (