From b5c87ab876cfdef7f71714de340d27599bcae2cd Mon Sep 17 00:00:00 2001 From: Steve Ruiz Date: Thu, 11 Apr 2024 16:31:21 +0100 Subject: [PATCH] Performance measurement tool (for unit tests) (#3447) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds a micro benchmarking utility. We can use it in our jest tests or in random scripts, though given the other requirements of our library, benchmarking. Screenshot 2024-04-11 at 2 44 23 PM ## What this isn't This is not benchmarking. The speeds etc are based on your machine. ## What this is This is a tool for measuring / comparing different implementations etc. Some things run much faster than others. ### Change Type - [x] `sdk` - [x] `internal` --- .../src/test/perf/PerformanceMeasurer.ts | 156 ++++++++++++++ packages/tldraw/src/test/perf/perf.test.ts | 191 ++++++++++++++++++ 2 files changed, 347 insertions(+) create mode 100644 packages/tldraw/src/test/perf/PerformanceMeasurer.ts create mode 100644 packages/tldraw/src/test/perf/perf.test.ts diff --git a/packages/tldraw/src/test/perf/PerformanceMeasurer.ts b/packages/tldraw/src/test/perf/PerformanceMeasurer.ts new file mode 100644 index 000000000..b07329a3d --- /dev/null +++ b/packages/tldraw/src/test/perf/PerformanceMeasurer.ts @@ -0,0 +1,156 @@ +const now = () => { + const hrTime = process.hrtime() + return hrTime[0] * 1000 + hrTime[1] / 1000000 +} + +export class PerformanceMeasurer { + private setupFn?: () => void + private beforeFns: (() => void)[] = [] + private fns: (() => void)[] = [] + private afterFns: (() => void)[] = [] + private teardownFn?: () => void + private warmupIterations = 0 + private iterations = 0 + + total = 0 + average = 0 + cold = 0 + fastest = Infinity + slowest = -Infinity + + totalStart = 0 + totalEnd = 0 + totalTime = 0 + + constructor( + public name: string, + opts = {} as { + warmupIterations?: number + iterations?: number + } + ) { + const { warmupIterations = 0, iterations = 10 } = opts + this.warmupIterations = warmupIterations + this.iterations = iterations + } + + setup(cb: () => void) { + this.setupFn = cb + return this + } + + teardown(cb: () => void) { + this.teardownFn = cb + return this + } + + add(cb: () => void) { + this.fns.push(cb) + return this + } + + before(cb: () => void) { + this.beforeFns.push(cb) + return this + } + + after(cb: () => void) { + this.afterFns.push(cb) + return this + } + + run() { + const { fns, beforeFns, afterFns, warmupIterations, iterations } = this + + // Run the cold run + this.setupFn?.() + const a = now() + for (let k = 0; k < beforeFns.length; k++) { + beforeFns[k]() + } + for (let j = 0; j < fns.length; j++) { + fns[j]() + } + for (let l = 0; l < afterFns.length; l++) { + afterFns[l]() + } + const duration = now() - a + this.cold = duration + this.teardownFn?.() + + // Run all of the warmup iterations + if (this.warmupIterations > 0) { + this.setupFn?.() + for (let i = 0; i < warmupIterations; i++) { + for (let k = 0; k < beforeFns.length; k++) { + beforeFns[k]() + } + const _a = now() + for (let j = 0; j < fns.length; j++) { + fns[j]() + } + const _duration = now() - a + for (let l = 0; l < afterFns.length; l++) { + afterFns[l]() + } + } + this.teardownFn?.() + } + + this.totalStart = now() + + // Run all of the iterations and calculate average + if (this.iterations > 0) { + this.setupFn?.() + for (let i = 0; i < iterations; i++) { + for (let k = 0; k < beforeFns.length; k++) { + beforeFns[k]() + } + const a = now() + for (let j = 0; j < fns.length; j++) { + fns[j]() + } + const duration = now() - a + this.total += duration + this.fastest = Math.min(duration, this.fastest) + this.slowest = Math.max(duration, this.fastest) + for (let l = 0; l < afterFns.length; l++) { + afterFns[l]() + } + } + this.teardownFn?.() + } + + this.totalTime = now() - this.totalStart + if (iterations > 0) { + this.average = this.total / iterations + } + + return this + } + + report() { + return PerformanceMeasurer.Table(this) + } + + static Table(...ps: PerformanceMeasurer[]) { + const table: Record> = {} + const fastest = ps.map((p) => p.average).reduce((a, b) => Math.min(a, b)) + const totalFastest = ps.map((p) => p.totalTime).reduce((a, b) => Math.min(a, b)) + + ps.forEach( + (p) => + (table[p.name] = { + ['Runs']: p.warmupIterations + p.iterations, + ['Cold']: Number(p.cold.toFixed(2)), + ['Slowest']: Number(p.slowest.toFixed(2)), + ['Fastest']: Number(p.fastest.toFixed(2)), + ['Average']: Number(p.average.toFixed(2)), + ['Slower (Avg)']: Number((p.average / fastest).toFixed(2)), + ['Slower (All)']: Number((p.totalTime / totalFastest).toFixed(2)), + }) + ) + // eslint-disable-next-line no-console + console.table(table) + } +} diff --git a/packages/tldraw/src/test/perf/perf.test.ts b/packages/tldraw/src/test/perf/perf.test.ts new file mode 100644 index 000000000..88593cecc --- /dev/null +++ b/packages/tldraw/src/test/perf/perf.test.ts @@ -0,0 +1,191 @@ +import { TLShapePartial, createShapeId } from '@tldraw/editor' +import { TestEditor } from '../TestEditor' +import { PerformanceMeasurer } from './PerformanceMeasurer' + +let editor = new TestEditor() + +jest.useRealTimers() + +describe.skip('Example perf tests', () => { + it('measures Editor.createShape vs Editor.createShapes', () => { + const withCreateShape = new PerformanceMeasurer('Create 200 shapes using Editor.createShape', { + warmupIterations: 10, + iterations: 10, + }) + .before(() => { + editor = new TestEditor() + }) + .add(() => { + for (let i = 0; i < 200; i++) { + editor.createShape({ + type: 'geo', + x: (i % 10) * 220, + y: Math.floor(i / 10) * 220, + props: { w: 200, h: 200 }, + }) + } + }) + + const withCreateShapes = new PerformanceMeasurer( + 'Create 200 shapes using Editor.createShapes', + { + warmupIterations: 10, + iterations: 10, + } + ) + .before(() => { + editor = new TestEditor() + }) + .add(() => { + const shapesToCreate: TLShapePartial[] = [] + for (let i = 0; i < 200; i++) { + shapesToCreate.push({ + id: createShapeId(), + type: 'geo', + x: (i % 10) * 220, + y: Math.floor(i / 10) * 220, + props: { w: 200, h: 200 }, + }) + } + + editor.createShapes(shapesToCreate) + }) + + withCreateShape.run() + withCreateShapes.run() + + PerformanceMeasurer.Table(withCreateShape, withCreateShapes) + + expect(withCreateShape.average).toBeGreaterThan(withCreateShapes.average) + }, 10000) + + it('measures Editor.updateShape vs Editor.updateShapes', () => { + const ids = Array.from(Array(200)).map(() => createShapeId()) + + const withUpdateShape = new PerformanceMeasurer('Update 200 shapes using Editor.updateShape', { + warmupIterations: 10, + iterations: 10, + }) + .before(() => { + editor = new TestEditor() + for (let i = 0; i < 200; i++) { + editor.createShape({ + type: 'geo', + id: ids[i], + x: (i % 10) * 220, + y: Math.floor(i / 10) * 220, + props: { w: 200, h: 200 }, + }) + } + }) + .add(() => { + for (let i = 0; i < 200; i++) { + editor.updateShape({ + type: 'geo', + id: ids[i], + x: (i % 10) * 220 + 1, + props: { + w: 201, + }, + }) + } + }) + + const withUpdateShapes = new PerformanceMeasurer( + 'Update 200 shapes using Editor.updateShapes', + { + warmupIterations: 10, + iterations: 10, + } + ) + .before(() => { + editor = new TestEditor() + for (let i = 0; i < 200; i++) { + editor.createShape({ + id: ids[i], + type: 'geo', + x: (i % 10) * 220, + y: Math.floor(i / 10) * 220, + props: { w: 200, h: 200 }, + }) + } + }) + .add(() => { + const shapesToUpdate: TLShapePartial[] = [] + for (let i = 0; i < 200; i++) { + shapesToUpdate.push({ + id: ids[i], + type: 'geo', + x: (i % 10) * 220 + 1, + props: { + w: 201, + }, + }) + } + editor.updateShapes(shapesToUpdate) + }) + + withUpdateShape.run() + withUpdateShapes.run() + + PerformanceMeasurer.Table(withUpdateShape, withUpdateShapes) + }, 10000) + + it('Measures rendering shapes', () => { + const renderingShapes = new PerformanceMeasurer('Measure rendering bounds with 100 shapes', { + warmupIterations: 10, + iterations: 20, + }) + .before(() => { + editor = new TestEditor() + const shapesToCreate: TLShapePartial[] = [] + for (let i = 0; i < 100; i++) { + shapesToCreate.push({ + id: createShapeId(), + type: 'geo', + x: (i % 10) * 220, + y: Math.floor(i / 10) * 220, + props: { w: 200, h: 200 }, + }) + } + editor.createShapes(shapesToCreate) + }) + .add(() => { + editor.getRenderingShapes() + }) + .after(() => { + const shape = editor.getCurrentPageShapes()[0] + editor.updateShape({ ...shape, x: shape.x + 1 }) + }) + .run() + + const renderingShapes2 = new PerformanceMeasurer('Measure rendering bounds with 200 shapes', { + warmupIterations: 10, + iterations: 20, + }) + .before(() => { + editor = new TestEditor() + const shapesToCreate: TLShapePartial[] = [] + for (let i = 0; i < 200; i++) { + shapesToCreate.push({ + id: createShapeId(), + type: 'geo', + x: (i % 10) * 220, + y: Math.floor(i / 10) * 220, + props: { w: 200, h: 200 }, + }) + } + editor.createShapes(shapesToCreate) + }) + .add(() => { + editor.getRenderingShapes() + }) + .after(() => { + const shape = editor.getCurrentPageShapes()[0] + editor.updateShape({ ...shape, x: shape.x + 1 }) + }) + .run() + + PerformanceMeasurer.Table(renderingShapes, renderingShapes2) + }, 10000) +})