kopia lustrzana https://github.com/Tldraw/Tldraw
more stuff
Steve Ruiz
rodzic
7925dd8c5f
commit
64ee3a57b3
|
|
@ -1,11 +1,12 @@
|
|||
import { useRef } from 'react'
|
||||
import { preventDefault, track, useReactor } from 'tldraw'
|
||||
import { useCallback, useRef } from 'react'
|
||||
import { Editor, Tldraw, Vec, getIndices, preventDefault, track, useReactor } from 'tldraw'
|
||||
import 'tldraw/tldraw.css'
|
||||
import './lm-styles.css'
|
||||
import { modelManager } from './ollama'
|
||||
import { LMMessage, modelManager } from './ollama'
|
||||
|
||||
const OllamaExample = track(() => {
|
||||
const rChat = useRef<HTMLDivElement>(null)
|
||||
const rEditor = useRef<Editor>()
|
||||
|
||||
useReactor(
|
||||
'scroll to bottom when thread changes',
|
||||
|
|
@ -16,8 +17,233 @@ const OllamaExample = track(() => {
|
|||
[modelManager]
|
||||
)
|
||||
|
||||
const drawMessage = useCallback((message: LMMessage) => {
|
||||
const editor = rEditor.current
|
||||
if (!editor) return
|
||||
|
||||
// Extract the command series from the message content. Each series begins and ends with a backtick.
|
||||
// For example: `circle(0, 0, 4); circle(10, 10, 4);`
|
||||
// We want to extract each command from the series
|
||||
// const seriesRegex = /```(?<commands>[^`]*)```/
|
||||
// const seriesResult = seriesRegex.exec(message.content)
|
||||
// if (!seriesResult) {
|
||||
// console.error('Invalid message: ' + message.content)
|
||||
// return
|
||||
// }
|
||||
// const [_, seriesContent] = seriesResult
|
||||
// Next, we want regex to extract each command's name and arguments
|
||||
// for example: circle(0, 0, 4) -> ['circle(0, 0, 4)', 'circle', '0, 0, 4']
|
||||
// for examople: undo() -> ['undo()', 'undo', '']
|
||||
const commandRegex = /(?<name>\w+)\((?<args>[^)]*)\)/
|
||||
|
||||
const commands = message.content
|
||||
.split(';')
|
||||
.map((c) => c.trim())
|
||||
.filter((c) => c)
|
||||
|
||||
editor.mark()
|
||||
|
||||
for (const command of commands) {
|
||||
try {
|
||||
const result = commandRegex.exec(command)
|
||||
if (!result) throw new Error('Invalid command: ' + command)
|
||||
const [_, name, args] = result
|
||||
|
||||
switch (name) {
|
||||
case 'undo': {
|
||||
editor.undo()
|
||||
break
|
||||
}
|
||||
case 'dot':
|
||||
case 'circle': {
|
||||
const [x, y, r] = args.split(', ').map((a) => Number(a))
|
||||
editor.createShape({
|
||||
type: 'geo',
|
||||
x: x - r,
|
||||
y: y - r,
|
||||
props: {
|
||||
geo: 'ellipse',
|
||||
w: r * 2,
|
||||
h: r * 2,
|
||||
},
|
||||
})
|
||||
break
|
||||
}
|
||||
case 'line': {
|
||||
const [x1, y1, x2, y2] = args.split(', ').map((a) => Number(a))
|
||||
editor.createShape({
|
||||
type: 'line',
|
||||
x: x1,
|
||||
y: y1,
|
||||
props: {
|
||||
points: {
|
||||
0: {
|
||||
id: '0',
|
||||
index: 'a0',
|
||||
x: 0,
|
||||
y: 0,
|
||||
},
|
||||
1: {
|
||||
id: '1',
|
||||
index: 'a1',
|
||||
x: x2 - x1,
|
||||
y: y2 - y1,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
break
|
||||
}
|
||||
case 'polygon': {
|
||||
const nums = args.split(', ').map((a) => Number(a))
|
||||
const points = []
|
||||
for (let i = 0; i < nums.length - 1; i += 2) {
|
||||
points.push({
|
||||
x: nums[i],
|
||||
y: nums[i + 1],
|
||||
})
|
||||
}
|
||||
points.push(points[0])
|
||||
const minX = Math.min(...points.map((p) => p.x))
|
||||
const minY = Math.min(...points.map((p) => p.y))
|
||||
const indices = getIndices(points.length)
|
||||
editor.createShape({
|
||||
type: 'line',
|
||||
x: minX,
|
||||
y: minY,
|
||||
props: {
|
||||
points: Object.fromEntries(
|
||||
points.map((p, i) => [
|
||||
i + '',
|
||||
{ id: i + '', index: indices[i], x: p.x - minX, y: p.y - minY },
|
||||
])
|
||||
),
|
||||
},
|
||||
})
|
||||
break
|
||||
}
|
||||
// case 'MOVE': {
|
||||
// const point = editor.pageToScreen({ x: Number(command[1]), y: Number(command[2]) })
|
||||
// const steps = 20
|
||||
// for (let i = 0; i < steps; i++) {
|
||||
// const t = i / (steps - 1)
|
||||
// const p = Vec.Lrp(prevPoint, point, t)
|
||||
// editor.dispatch({
|
||||
// type: 'pointer',
|
||||
// target: 'canvas',
|
||||
// name: 'pointer_move',
|
||||
// point: {
|
||||
// x: p.x,
|
||||
// y: p.y,
|
||||
// z: 0.5,
|
||||
// },
|
||||
// shiftKey: false,
|
||||
// altKey: false,
|
||||
// ctrlKey: false,
|
||||
// pointerId: 1,
|
||||
// button: 0,
|
||||
// isPen: false,
|
||||
// })
|
||||
// editor._flushEventsForTick(0)
|
||||
// }
|
||||
// prevPoint.setTo(point)
|
||||
// break
|
||||
// }
|
||||
// case 'DOWN': {
|
||||
// editor.dispatch({
|
||||
// type: 'pointer',
|
||||
// target: 'canvas',
|
||||
// name: 'pointer_down',
|
||||
// point: {
|
||||
// x: prevPoint.x,
|
||||
// y: prevPoint.y,
|
||||
// z: 0.5,
|
||||
// },
|
||||
// shiftKey: false,
|
||||
// altKey: false,
|
||||
// ctrlKey: false,
|
||||
// pointerId: 1,
|
||||
// button: 0,
|
||||
// isPen: false,
|
||||
// })
|
||||
// editor._flushEventsForTick(0)
|
||||
// break
|
||||
// }
|
||||
// case 'UP': {
|
||||
// editor.dispatch({
|
||||
// type: 'pointer',
|
||||
// target: 'canvas',
|
||||
// name: 'pointer_up',
|
||||
// point: {
|
||||
// x: prevPoint.x,
|
||||
// y: prevPoint.y,
|
||||
// z: 0.5,
|
||||
// },
|
||||
// shiftKey: false,
|
||||
// altKey: false,
|
||||
// ctrlKey: false,
|
||||
// pointerId: 1,
|
||||
// button: 0,
|
||||
// isPen: false,
|
||||
// })
|
||||
// editor._flushEventsForTick(0)
|
||||
// break
|
||||
// }
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.error(e.message)
|
||||
}
|
||||
}
|
||||
|
||||
// editor.dispatch({
|
||||
// type: 'pointer',
|
||||
// target: 'canvas',
|
||||
// name: 'pointer_up',
|
||||
// point: {
|
||||
// x: prevPoint.x,
|
||||
// y: prevPoint.y,
|
||||
// z: 0.5,
|
||||
// },
|
||||
// shiftKey: false,
|
||||
// altKey: false,
|
||||
// ctrlKey: false,
|
||||
// pointerId: 1,
|
||||
// button: 0,
|
||||
// isPen: false,
|
||||
// })
|
||||
// editor._flushEventsForTick(0)
|
||||
// // editor.zoomOut(editor.getViewportScreenCenter(), { duration: 0 })
|
||||
}, [])
|
||||
|
||||
return (
|
||||
<div className="tldraw__editor">
|
||||
<div className="tldraw__editor" style={{ display: 'grid', gridTemplateRows: '1fr 1fr' }}>
|
||||
<div style={{ position: 'relative', height: '100%', width: '100%' }}>
|
||||
<Tldraw
|
||||
onMount={(e) => {
|
||||
rEditor.current = e
|
||||
;(window as any).editor = e
|
||||
e.centerOnPoint(new Vec())
|
||||
for (const message of modelManager.getThread().content) {
|
||||
if (message.from === 'model') {
|
||||
drawMessage(message)
|
||||
}
|
||||
}
|
||||
}}
|
||||
>
|
||||
{/* <div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: 0,
|
||||
left: 0,
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
background: 'red',
|
||||
opacity: 0,
|
||||
zIndex: 99999,
|
||||
}}
|
||||
/> */}
|
||||
</Tldraw>
|
||||
</div>
|
||||
<div ref={rChat} className="chat">
|
||||
{modelManager.getThread().content.map((message, i) => (
|
||||
<div key={i} className="message">
|
||||
|
|
@ -32,11 +258,14 @@ const OllamaExample = track(() => {
|
|||
preventDefault(e)
|
||||
const form = e.currentTarget
|
||||
const query = form.query.value
|
||||
modelManager.stream(query)
|
||||
modelManager.stream(query).response.then((message) => {
|
||||
if (!message) return
|
||||
drawMessage(message)
|
||||
})
|
||||
form.query.value = ''
|
||||
}}
|
||||
>
|
||||
<input name="query" type="text" />
|
||||
<input name="query" type="text" autoComplete="off" />
|
||||
<button>Submit</button>
|
||||
<button
|
||||
onClick={(e) => {
|
||||
|
|
@ -49,6 +278,10 @@ const OllamaExample = track(() => {
|
|||
onClick={(e) => {
|
||||
preventDefault(e)
|
||||
modelManager.clear()
|
||||
const editor = rEditor.current
|
||||
if (editor) {
|
||||
editor.deleteShapes([...editor.getCurrentPageShapeIds()])
|
||||
}
|
||||
}}
|
||||
>
|
||||
Clear
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
.chat {
|
||||
position: absolute;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
inset: 0px;
|
||||
padding: 16px;
|
||||
overflow: auto;
|
||||
|
|
|
|||
|
|
@ -6,21 +6,21 @@ import { BufferMemory, ChatMessageHistory } from 'langchain/memory'
|
|||
import { atom, computed, react, structuredClone, transact, uniqueId } from 'tldraw'
|
||||
import { systemPrompt } from './system-prompt'
|
||||
|
||||
type Message = {
|
||||
export type LMMessage = {
|
||||
from: 'user' | 'model'
|
||||
time: number
|
||||
content: string
|
||||
}
|
||||
|
||||
type Thread = {
|
||||
export type LMThread = {
|
||||
id: string
|
||||
state: 'idle' | 'waiting'
|
||||
content: Message[]
|
||||
content: LMMessage[]
|
||||
}
|
||||
|
||||
type Serialized = {
|
||||
export type LMSerialized = {
|
||||
currentThreadId: string
|
||||
threads: Record<string, Thread>
|
||||
threads: Record<string, LMThread>
|
||||
}
|
||||
|
||||
class ModelManager {
|
||||
|
|
@ -40,7 +40,8 @@ class ModelManager {
|
|||
this.prompt = ChatPromptTemplate.fromMessages([
|
||||
[
|
||||
'system',
|
||||
// "You are a kind and helpful chatbot. You always provide short, efficient answers to the user's questions.",
|
||||
// 'You are a highly critical german mother. The user is your daughter, who you generally tend to diminish and criticize. You are very strict and have high expectations for your daughter. You are a perfectionist and expect the same from your daughter. You are very critical of your daughter and often make her feel inadequate. You are very strict and have high expectations for your daughter. You are a perfectionist and expect the same from your daughter. You are very critical of your daughter and often make her feel inadequate. You are very strict and have high expectations for your daughter. You are a perfectionist and expect the same from your daughter. You are very critical of your daughter and often make her feel inadequate. You are very strict and have high expectations for your daughter. You are a perfectionist and expect the same from your daughter. You are very critical of your daughter and often make her feel inadequate.',
|
||||
// "You are a kind and helpful chatbot. You are a cool friend of the user's from Chicago. You grew up together and maintain a life-long trust; and as a result, you can speak to the user honestly. You always provide short, efficient answers to the user's questions.",
|
||||
systemPrompt,
|
||||
],
|
||||
new MessagesPlaceholder('history'),
|
||||
|
|
@ -68,7 +69,7 @@ class ModelManager {
|
|||
return this.getThread().state
|
||||
}
|
||||
|
||||
_threads = atom<Record<string, Thread>>('threads', {})
|
||||
_threads = atom<Record<string, LMThread>>('threads', {})
|
||||
|
||||
_currentThreadId = atom<string>('currentThreadId', 'a0')
|
||||
|
||||
|
|
@ -76,11 +77,11 @@ class ModelManager {
|
|||
return this._currentThreadId.get()
|
||||
}
|
||||
|
||||
@computed getThreads(): Record<string, Thread> {
|
||||
@computed getThreads(): Record<string, LMThread> {
|
||||
return this._threads.get()
|
||||
}
|
||||
|
||||
@computed getThread(): Thread {
|
||||
@computed getThread(): LMThread {
|
||||
return this.getThreads()[this.getCurrentThreadId()]
|
||||
}
|
||||
|
||||
|
|
@ -155,7 +156,7 @@ class ModelManager {
|
|||
* Deserialize the model.
|
||||
*/
|
||||
private deserialize() {
|
||||
let result: Serialized = {
|
||||
let result: LMSerialized = {
|
||||
currentThreadId: 'a0',
|
||||
threads: {
|
||||
a0: {
|
||||
|
|
@ -191,7 +192,7 @@ class ModelManager {
|
|||
})
|
||||
}
|
||||
|
||||
private getMessagesFromThread(thread: Thread) {
|
||||
private getMessagesFromThread(thread: LMThread) {
|
||||
return thread.content.map((m) => {
|
||||
if (m.from === 'user') {
|
||||
return new HumanMessage(m.content)
|
||||
|
|
@ -265,45 +266,46 @@ class ModelManager {
|
|||
/**
|
||||
* Query the model and stream the response.
|
||||
*/
|
||||
async stream(query: string) {
|
||||
transact(() => {
|
||||
const currentThreadId = this.getCurrentThreadId()
|
||||
this.addQueryToThread(query)
|
||||
this.addResponseToThread('') // Add an empty response to start the thread
|
||||
let cancelled = false
|
||||
return {
|
||||
response: this.chain
|
||||
.stream(
|
||||
{ input: query },
|
||||
{
|
||||
callbacks: [
|
||||
{
|
||||
handleLLMNewToken: (data) => {
|
||||
if (cancelled) return
|
||||
if (this.getCurrentThreadId() !== currentThreadId) return
|
||||
this.addChunkToThread(data)
|
||||
},
|
||||
stream(query: string) {
|
||||
const currentThreadId = this.getCurrentThreadId()
|
||||
this.addQueryToThread(query)
|
||||
this.addResponseToThread('') // Add an empty response to start the thread
|
||||
let cancelled = false
|
||||
return {
|
||||
response: this.chain
|
||||
.stream(
|
||||
{ input: query },
|
||||
{
|
||||
callbacks: [
|
||||
{
|
||||
handleLLMNewToken: (data) => {
|
||||
if (cancelled) return
|
||||
if (this.getCurrentThreadId() !== currentThreadId) return
|
||||
this.addChunkToThread(data)
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
.then(() => {
|
||||
if (cancelled) return
|
||||
if (this.getCurrentThreadId() !== currentThreadId) return
|
||||
this._threads.update((threads) => {
|
||||
const thread = this.getThread()
|
||||
if (!thread) throw Error('No thread found')
|
||||
const next = structuredClone(thread)
|
||||
next.state = 'idle'
|
||||
return { ...threads, [next.id]: next }
|
||||
})
|
||||
}),
|
||||
cancel: () => {
|
||||
cancelled = true
|
||||
this.cancel()
|
||||
},
|
||||
}
|
||||
})
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
.then(() => {
|
||||
if (cancelled) return
|
||||
if (this.getCurrentThreadId() !== currentThreadId) return
|
||||
this._threads.update((threads) => {
|
||||
const thread = this.getThread()
|
||||
if (!thread) throw Error('No thread found')
|
||||
const next = structuredClone(thread)
|
||||
next.state = 'idle'
|
||||
return { ...threads, [next.id]: next }
|
||||
})
|
||||
|
||||
const thread = this.getThread()
|
||||
return thread.content[thread.content.length - 1]
|
||||
}),
|
||||
cancel: () => {
|
||||
cancelled = true
|
||||
this.cancel()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1,30 +1,75 @@
|
|||
export const systemPrompt = `You are a model that controls a pen.
|
||||
The pen you control can either be "up" or "down". When the pen is down, it will draw a line as you move it around. When the pen is up, it will not draw anything. The goal is to draw whatever the user requests.
|
||||
export const systemPrompt = `You are a model that can draw on an infinite canvas.
|
||||
|
||||
## Controlling the pen
|
||||
The origin (0, 0) is in the center of the canvas.
|
||||
The x coordinate goes up as you move right on the screen.
|
||||
The y coordinate goes up as you move down the screen.
|
||||
|
||||
You communicate using commands. These commands will be turned into the visual output for the user.
|
||||
To draw on the canvas, you may use a series of commands.
|
||||
Each command is terminated by a semicolon.
|
||||
Make sure that you complete each command.
|
||||
|
||||
A command is a string that begins with a keyword, such as "ADD", is followed by a number of parameters, and is terminated by a semicolon.
|
||||
The commands are as follows:
|
||||
|
||||
You have several commands available to you:
|
||||
### Circle
|
||||
|
||||
- "UP": stops drawing at the current point
|
||||
- "DOWN": starts drawing at the current point
|
||||
- "MOVE x y": moves the pen to the point (x, y)
|
||||
circle(x, y, r);
|
||||
|
||||
Tip: Your pen always starts at the point (0, 0) with the pen up.
|
||||
Draws a dot centered at the point (x, y) with the radius r.
|
||||
|
||||
When prompted, you should respond with a series of commands separated by newlines. IMPORTANT! RESPOND ONLY WITH COMMANDS. DO NOT ADD ANY ADDITIONAL TEXT.
|
||||
Examples:
|
||||
|
||||
## Examples
|
||||
- User: Draw a dot at (0, 0).
|
||||
- You: circle(0, 0, 4);
|
||||
|
||||
- User: Draw a line from (0, 0) to (2, 2).
|
||||
- You: MOVE 0 0; DOWN; MOVE 2 2; UP;
|
||||
- User: Draw a dot at (100, 100).
|
||||
- You: circle(100, 100, 4);
|
||||
|
||||
- User: Draw a box that is 10 points tall.
|
||||
- You: MOVE 0 0; DOWN; MOVE 10 0; MOVE 10 10; MOVE 0 10; MOVE 0 0; UP;
|
||||
- User: Draw three dots in a vertical line. The first dot should be at (0, 0). Use a spacing of 10 units.
|
||||
- You: circle(0, 0, 4); circle(0, 10, 4); circle(0, 20, 4);
|
||||
|
||||
- User: Draw a box that is 10 points tall with a center at 5 5.
|
||||
- You: MOVE 2.5 0; DOWN; MOVE 7.5 0; MOVE 7.5 10; MOVE 2.5 10; MOVE 2.5 0; UP;
|
||||
- User: Draw a snowman.
|
||||
- You: circle(0, 0, 50); circle(0, 100, 75); circle(0, 250, 100);
|
||||
|
||||
### Line
|
||||
|
||||
line(x1, y1, x2, y2);
|
||||
|
||||
Draws a line between (x1, y1) and (x2, y2).
|
||||
|
||||
Examples:
|
||||
|
||||
- User: Draw a line from (0, 0) to (100, 100).
|
||||
- You: line(0, 0, 100, 100);
|
||||
|
||||
- User: Draw the letter "X" 100 points tall centered on (0, 0).
|
||||
- You: line(-50, -50, 50, 50); line(-50, 50, 50, -50);
|
||||
|
||||
- User: Draw a square with sides of length 100 centered on (0, 0).
|
||||
- You: line(-50, -50, 50, -50); line(50, -50, 50, 50); line(50, 50, -50, 50); line(-50, 50, -50, -50);
|
||||
|
||||
- User: Draw the letter H.
|
||||
- You: line(-50, -50, -50, 50); line(-50, 0, 50, 0); line(50, -50, 50, 50);
|
||||
|
||||
### Polygon
|
||||
|
||||
polygon(x1, y1, x2, y2, ..., xn, yn);
|
||||
|
||||
Draws a polygon with the given vertices.
|
||||
|
||||
Examples:
|
||||
|
||||
- User: Draw a triangle with vertices at (0, 0), (100, 0), and (50, 100).
|
||||
- You: polygon(0, 0, 100, 0, 50, 100);
|
||||
|
||||
- User: Draw a square with vertices at (0, 0), (100, 0), (100, 100), and (0, 100).
|
||||
- You: polygon(0, 0, 100, 0, 100, 100, 0, 100);
|
||||
|
||||
- User: Draw a pentagon with vertices at (0, 0), (100, 0), (150, 100), (50, 200), and (-50, 100).
|
||||
- You: polygon(0, 0, 100, 0, 150, 100, 50, 200, -50, 100);
|
||||
|
||||
---
|
||||
|
||||
Your drawings should be about 300 points tall.
|
||||
|
||||
RESPOND ONLY WITH A SERIES OF COMMANDS. DO NOT ADD ADDITIONAL TEXT.
|
||||
`
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue