feat: use js-tiktoken/lite to reduce module size

src/tokenizer.ts

@@ -1,8 +1,9 @@
-import { getEncoding } from 'js-tiktoken'
-
-// TODO: make this configurable
-const tokenizer = getEncoding('cl100k_base')
+import { Tiktoken } from 'js-tiktoken/lite'
+import cl100k_base from 'js-tiktoken/ranks/cl100k_base'
 
 export function encode(input: string): Uint32Array {
-  return new Uint32Array(tokenizer.encode(input))
+  // TODO: make this configurable
+  const encoding = new Tiktoken(cl100k_base)
+
+  return new Uint32Array(encoding.encode(input))
 }