mirror
/
chatgpt-api
kopia lustrzana https://github.com/transitive-bullshit/chatgpt-api
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

feat: switch js-tiktoken to load tokenizer ranks eagerly

old-agentic-v1^2
Travis Fischer 2023-06-28 20:16:56 -07:00
rodzic 59c0189ecd
commit 18e015029c
4 zmienionych plików z 163 dodań i 183 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

6
legacy/package.json
Wyświetl plik

@ -66,7 +66,7 @@
"replicate": "^0.12.3",
"ts-dedent": "^2.2.0",
"zod": "^3.21.4",
"zod-to-json-schema": "^3.21.2",
"zod-to-json-schema": "^3.21.3",
"zod-validation-error": "^1.3.1"
},
"devDependencies": {
@ -88,7 +88,7 @@
"husky": "^8.0.3",
"ioredis": "^5.3.2",
"keyv": "^4.5.2",
"lint-staged": "^13.2.2",
"lint-staged": "^13.2.3",
"npm-run-all": "^4.1.5",
"p-memoize": "^7.1.1",
"prettier": "^2.8.8",
@ -99,7 +99,7 @@
"type-fest": "^3.12.0",
"typedoc": "^0.24.8",
"typedoc-plugin-markdown": "4.0.0-next.16",
"typescript": "^5.1.3"
"typescript": "^5.1.6"
},
"lint-staged": {
"*.{ts,tsx}": [

294
legacy/pnpm-lock.yaml
Wyświetl plik

@ -87,8 +87,8 @@ dependencies:
specifier: ^3.21.4
version: 3.21.4
zod-to-json-schema:
specifier: ^3.21.2
version: 3.21.2(zod@3.21.4)
specifier: ^3.21.3
version: 3.21.3(zod@3.21.4)
zod-validation-error:
specifier: ^1.3.1
version: 1.3.1(zod@3.21.4)
@ -111,10 +111,10 @@ devDependencies:
version: 10.0.15
'@typescript-eslint/eslint-plugin':
specifier: ^5.60.1
version: 5.60.1(@typescript-eslint/parser@5.60.1)(eslint@8.43.0)(typescript@5.1.3)
version: 5.60.1(@typescript-eslint/parser@5.60.1)(eslint@8.43.0)(typescript@5.1.6)
'@typescript-eslint/parser':
specifier: ^5.60.1
version: 5.60.1(eslint@8.43.0)(typescript@5.1.3)
version: 5.60.1(eslint@8.43.0)(typescript@5.1.6)
ava:
specifier: ^5.3.1
version: 5.3.1
@ -149,8 +149,8 @@ devDependencies:
specifier: ^4.5.2
version: 4.5.2
lint-staged:
specifier: ^13.2.2
version: 13.2.2
specifier: ^13.2.3
version: 13.2.3
npm-run-all:
specifier: ^4.1.5
version: 4.1.5
@ -168,7 +168,7 @@ devDependencies:
version: 15.2.0
tsup:
specifier: 7.1.0
version: 7.1.0(typescript@5.1.3)
version: 7.1.0(typescript@5.1.6)
tsx:
specifier: ^3.12.7
version: 3.12.7
@ -177,16 +177,21 @@ devDependencies:
version: 3.12.0
typedoc:
specifier: ^0.24.8
version: 0.24.8(typescript@5.1.3)
version: 0.24.8(typescript@5.1.6)
typedoc-plugin-markdown:
specifier: 4.0.0-next.16
version: 4.0.0-next.16(prettier@2.8.8)(typedoc@0.24.8)
typescript:
specifier: ^5.1.3
version: 5.1.3
specifier: ^5.1.6
version: 5.1.6
packages:
/@aashutoshrathi/word-wrap@1.2.6:
resolution: {integrity: sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==}
engines: {node: '>=0.10.0'}
dev: true
/@agentic/midjourney-fetch@1.0.1:
resolution: {integrity: sha512-CUGNh3YJfbAPrOKcmqOOWuWQrkgVY+pWbCGhZkq74Jlc6oP8Jb+/K61V7Jhf/liYgYopbrpoDDue+ZKqijW/2A==}
engines: {node: '>=18', pnpm: '>=8'}
@ -326,7 +331,7 @@ packages:
resolution: {integrity: sha512-BDXFbYOJzT/NBEtp71cvsrGPwGAMGRB/349rwKuoxNSiKjPraNNnlK6MIIabViCjqZugu6j+xeMDlEkWdHHJSg==}
dependencies:
'@esbuild-kit/core-utils': 3.1.0
get-tsconfig: 4.6.0
get-tsconfig: 4.6.2
dev: true
/@esbuild-kit/core-utils@3.1.0:
@ -340,7 +345,7 @@ packages:
resolution: {integrity: sha512-Qwfvj/qoPbClxCRNuac1Du01r9gvNOT+pMYtJDapfB1eoGN1YlJ1BixLyL9WVENRx5RXgNLdfYdx/CuswlGhMw==}
dependencies:
'@esbuild-kit/core-utils': 3.1.0
get-tsconfig: 4.6.0
get-tsconfig: 4.6.2
dev: true
/@esbuild/android-arm64@0.17.19:
@ -352,8 +357,8 @@ packages:
dev: true
optional: true
/@esbuild/android-arm64@0.18.4:
resolution: {integrity: sha512-yQVgO+V307hA2XhzELQ6F91CBGX7gSnlVGAj5YIqjQOxThDpM7fOcHT2YLJbE6gNdPtgRSafQrsK8rJ9xHCaZg==}
/@esbuild/android-arm64@0.18.10:
resolution: {integrity: sha512-ynm4naLbNbK0ajf9LUWtQB+6Vfg1Z/AplArqr4tGebC00Z6m9Y91OVIcjDa461wGcZwcaHYaZAab4yJxfhisTQ==}
engines: {node: '>=12'}
cpu: [arm64]
os: [android]
@ -370,8 +375,8 @@ packages:
dev: true
optional: true
/@esbuild/android-arm@0.18.4:
resolution: {integrity: sha512-yKmQC9IiuvHdsNEbPHSprnMHg6OhL1cSeQZLzPpgzJBJ9ppEg9GAZN8MKj1TcmB4tZZUrq5xjK7KCmhwZP8iDA==}
/@esbuild/android-arm@0.18.10:
resolution: {integrity: sha512-3KClmVNd+Fku82uZJz5C4Rx8m1PPmWUFz5Zkw8jkpZPOmsq+EG1TTOtw1OXkHuX3WczOFQigrtf60B1ijKwNsg==}
engines: {node: '>=12'}
cpu: [arm]
os: [android]
@ -388,8 +393,8 @@ packages:
dev: true
optional: true
/@esbuild/android-x64@0.18.4:
resolution: {integrity: sha512-yLKXMxQg6sk1ntftxQ5uwyVgG4/S2E7UoOCc5N4YZW7fdkfRiYEXqm7CMuIfY2Vs3FTrNyKmSfNevIuIvJnMww==}
/@esbuild/android-x64@0.18.10:
resolution: {integrity: sha512-vFfXj8P9Yfjh54yqUDEHKzqzYuEfPyAOl3z7R9hjkwt+NCvbn9VMxX+IILnAfdImRBfYVItgSUsqGKhJFnBwZw==}
engines: {node: '>=12'}
cpu: [x64]
os: [android]
@ -406,8 +411,8 @@ packages:
dev: true
optional: true
/@esbuild/darwin-arm64@0.18.4:
resolution: {integrity: sha512-MVPEoZjZpk2xQ1zckZrb8eQuQib+QCzdmMs3YZAYEQPg+Rztk5pUxGyk8htZOC8Z38NMM29W+MqY9Sqo/sDGKw==}
/@esbuild/darwin-arm64@0.18.10:
resolution: {integrity: sha512-k2OJQ7ZxE6sVc91+MQeZH9gFeDAH2uIYALPAwTjTCvcPy9Dzrf7V7gFUQPYkn09zloWhQ+nvxWHia2x2ZLR0sQ==}
engines: {node: '>=12'}
cpu: [arm64]
os: [darwin]
@ -424,8 +429,8 @@ packages:
dev: true
optional: true
/@esbuild/darwin-x64@0.18.4:
resolution: {integrity: sha512-uEsRtYRUDsz7i2tXg/t/SyF+5gU1cvi9B6B8i5ebJgtUUHJYWyIPIesmIOL4/+bywjxsDMA/XrNFMgMffLnh5A==}
/@esbuild/darwin-x64@0.18.10:
resolution: {integrity: sha512-tnz/mdZk1L1Z3WpGjin/L2bKTe8/AKZpI8fcCLtH+gq8WXWsCNJSxlesAObV4qbtTl6pG5vmqFXfWUQ5hV8PAQ==}
engines: {node: '>=12'}
cpu: [x64]
os: [darwin]
@ -442,8 +447,8 @@ packages:
dev: true
optional: true
/@esbuild/freebsd-arm64@0.18.4:
resolution: {integrity: sha512-I8EOigqWnOHRin6Zp5Y1cfH3oT54bd7Sdz/VnpUNksbOtfp8IWRTH4pgkgO5jWaRQPjCpJcOpdRjYAMjPt8wXg==}
/@esbuild/freebsd-arm64@0.18.10:
resolution: {integrity: sha512-QJluV0LwBrbHnYYwSKC+K8RGz0g/EyhpQH1IxdoFT0nM7PfgjE+aS8wxq/KFEsU0JkL7U/EEKd3O8xVBxXb2aA==}
engines: {node: '>=12'}
cpu: [arm64]
os: [freebsd]
@ -460,8 +465,8 @@ packages:
dev: true
optional: true
/@esbuild/freebsd-x64@0.18.4:
resolution: {integrity: sha512-1bHfgMz/cNMjbpsYxjVgMJ1iwKq+NdDPlACBrWULD7ZdFmBQrhMicMaKb5CdmdVyvIwXmasOuF4r6Iq574kUTA==}
/@esbuild/freebsd-x64@0.18.10:
resolution: {integrity: sha512-Hi/ycUkS6KTw+U9G5PK5NoK7CZboicaKUSVs0FSiPNtuCTzK6HNM4DIgniH7hFaeuszDS9T4dhAHWiLSt/Y5Ng==}
engines: {node: '>=12'}
cpu: [x64]
os: [freebsd]
@ -478,8 +483,8 @@ packages:
dev: true
optional: true
/@esbuild/linux-arm64@0.18.4:
resolution: {integrity: sha512-J42vLHaYREyiBwH0eQE4/7H1DTfZx8FuxyWSictx4d7ezzuKE3XOkIvOg+SQzRz7T9HLVKzq2tvbAov4UfufBw==}
/@esbuild/linux-arm64@0.18.10:
resolution: {integrity: sha512-Nz6XcfRBOO7jSrVpKAyEyFOPGhySPNlgumSDhWAspdQQ11ub/7/NZDMhWDFReE9QH/SsCOCLQbdj0atAk/HMOQ==}
engines: {node: '>=12'}
cpu: [arm64]
os: [linux]
@ -496,8 +501,8 @@ packages:
dev: true
optional: true
/@esbuild/linux-arm@0.18.4:
resolution: {integrity: sha512-4XCGqM/Ay1LCXUBH59bL4JbSbbTK1K22dWHymWMGaEh2sQCDOUw+OQxozYV/YdBb91leK2NbuSrE2BRamwgaYw==}
/@esbuild/linux-arm@0.18.10:
resolution: {integrity: sha512-HfFoxY172tVHPIvJy+FHxzB4l8xU7e5cxmNS11cQ2jt4JWAukn/7LXaPdZid41UyTweqa4P/1zs201gRGCTwHw==}
engines: {node: '>=12'}
cpu: [arm]
os: [linux]
@ -514,8 +519,8 @@ packages:
dev: true
optional: true
/@esbuild/linux-ia32@0.18.4:
resolution: {integrity: sha512-4ksIqFwhq7OExty7Sl1n0vqQSCqTG4sU6i99G2yuMr28CEOUZ/60N+IO9hwI8sIxBqmKmDgncE1n5CMu/3m0IA==}
/@esbuild/linux-ia32@0.18.10:
resolution: {integrity: sha512-otMdmSmkMe+pmiP/bZBjfphyAsTsngyT9RCYwoFzqrveAbux9nYitDTpdgToG0Z0U55+PnH654gCH2GQ1aB6Yw==}
engines: {node: '>=12'}
cpu: [ia32]
os: [linux]
@ -532,8 +537,8 @@ packages:
dev: true
optional: true
/@esbuild/linux-loong64@0.18.4:
resolution: {integrity: sha512-bsWtoVHkGQgAsFXioDueXRiUIfSGrVkJjBBz4gcBJxXcD461cWFQFyu8Fxdj9TP+zEeqJ8C/O4LFFMBNi6Fscw==}
/@esbuild/linux-loong64@0.18.10:
resolution: {integrity: sha512-t8tjFuON1koxskzQ4VFoh0T5UDUMiLYjwf9Wktd0tx8AoK6xgU+5ubKOpWpcnhEQ2tESS5u0v6QuN8PX/ftwcQ==}
engines: {node: '>=12'}
cpu: [loong64]
os: [linux]
@ -550,8 +555,8 @@ packages:
dev: true
optional: true
/@esbuild/linux-mips64el@0.18.4:
resolution: {integrity: sha512-LRD9Fu8wJQgIOOV1o3nRyzrheFYjxA0C1IVWZ93eNRRWBKgarYFejd5WBtrp43cE4y4D4t3qWWyklm73Mrsd/g==}
/@esbuild/linux-mips64el@0.18.10:
resolution: {integrity: sha512-+dUkcVzcfEJHz3HEnVpIJu8z8Wdn2n/nWMWdl6FVPFGJAVySO4g3+XPzNKFytVFwf8hPVDwYXzVcu8GMFqsqZw==}
engines: {node: '>=12'}
cpu: [mips64el]
os: [linux]
@ -568,8 +573,8 @@ packages:
dev: true
optional: true
/@esbuild/linux-ppc64@0.18.4:
resolution: {integrity: sha512-jtQgoZjM92gauVRxNaaG/TpL3Pr4WcL3Pwqi9QgdrBGrEXzB+twohQiWNSTycs6lUygakos4mm2h0B9/SHveng==}
/@esbuild/linux-ppc64@0.18.10:
resolution: {integrity: sha512-sO3PjjxEGy+PY2qkGe2gwJbXdZN9wAYpVBZWFD0AwAoKuXRkWK0/zaMQ5ekUFJDRDCRm8x5U0Axaub7ynH/wVg==}
engines: {node: '>=12'}
cpu: [ppc64]
os: [linux]
@ -586,8 +591,8 @@ packages:
dev: true
optional: true
/@esbuild/linux-riscv64@0.18.4:
resolution: {integrity: sha512-7WaU/kRZG0VCV09Xdlkg6LNAsfU9SAxo6XEdaZ8ffO4lh+DZoAhGTx7+vTMOXKxa+r2w1LYDGxfJa2rcgagMRA==}
/@esbuild/linux-riscv64@0.18.10:
resolution: {integrity: sha512-JDtdbJg3yjDeXLv4lZYE1kiTnxv73/8cbPHY9T/dUKi8rYOM/k5b3W4UJLMUksuQ6nTm5c89W1nADsql6FW75A==}
engines: {node: '>=12'}
cpu: [riscv64]
os: [linux]
@ -604,8 +609,8 @@ packages:
dev: true
optional: true
/@esbuild/linux-s390x@0.18.4:
resolution: {integrity: sha512-D19ed0xreKQvC5t+ArE2njSnm18WPpE+1fhwaiJHf+Xwqsq+/SUaV8Mx0M27nszdU+Atq1HahrgCOZCNNEASUg==}
/@esbuild/linux-s390x@0.18.10:
resolution: {integrity: sha512-NLuSKcp8WckjD2a7z5kzLiCywFwBTMlIxDNuud1AUGVuwBBJSkuubp6cNjJ0p5c6CZaA3QqUGwjHJBiG1SoOFw==}
engines: {node: '>=12'}
cpu: [s390x]
os: [linux]
@ -622,8 +627,8 @@ packages:
dev: true
optional: true
/@esbuild/linux-x64@0.18.4:
resolution: {integrity: sha512-Rx3AY1sxyiO/gvCGP00nL69L60dfmWyjKWY06ugpB8Ydpdsfi3BHW58HWC24K3CAjAPSwxcajozC2PzA9JBS1g==}
/@esbuild/linux-x64@0.18.10:
resolution: {integrity: sha512-wj2KRsCsFusli+6yFgNO/zmmLslislAWryJnodteRmGej7ZzinIbMdsyp13rVGde88zxJd5vercNYK9kuvlZaQ==}
engines: {node: '>=12'}
cpu: [x64]
os: [linux]
@ -640,8 +645,8 @@ packages:
dev: true
optional: true
/@esbuild/netbsd-x64@0.18.4:
resolution: {integrity: sha512-AaShPmN9c6w1mKRpliKFlaWcSkpBT4KOlk93UfFgeI3F3cbjzdDKGsbKnOZozmYbE1izZKLmNJiW0sFM+A5JPA==}
/@esbuild/netbsd-x64@0.18.10:
resolution: {integrity: sha512-pQ9QqxEPI3cVRZyUtCoZxhZK3If+7RzR8L2yz2+TDzdygofIPOJFaAPkEJ5rYIbUO101RaiYxfdOBahYexLk5A==}
engines: {node: '>=12'}
cpu: [x64]
os: [netbsd]
@ -658,8 +663,8 @@ packages:
dev: true
optional: true
/@esbuild/openbsd-x64@0.18.4:
resolution: {integrity: sha512-tRGvGwou3BrvHVvF8HxTqEiC5VtPzySudS9fh2jBIKpLX7HCW8jIkW+LunkFDNwhslx4xMAgh0jAHsx/iCymaQ==}
/@esbuild/openbsd-x64@0.18.10:
resolution: {integrity: sha512-k8GTIIW9I8pEEfoOUm32TpPMgSg06JhL5DO+ql66aLTkOQUs0TxCA67Wi7pv6z8iF8STCGcNbm3UWFHLuci+ag==}
engines: {node: '>=12'}
cpu: [x64]
os: [openbsd]
@ -676,8 +681,8 @@ packages:
dev: true
optional: true
/@esbuild/sunos-x64@0.18.4:
resolution: {integrity: sha512-acORFDI95GKhmAnlH8EarBeuqoy/j3yxIU+FDB91H3+ZON+8HhTadtT450YkaMzX6lEWbhi+mjVUCj00M5yyOQ==}
/@esbuild/sunos-x64@0.18.10:
resolution: {integrity: sha512-vIGYJIdEI6d4JBucAx8py792G8J0GP40qSH+EvSt80A4zvGd6jph+5t1g+eEXcS2aRpgZw6CrssNCFZxTdEsxw==}
engines: {node: '>=12'}
cpu: [x64]
os: [sunos]
@ -694,8 +699,8 @@ packages:
dev: true
optional: true
/@esbuild/win32-arm64@0.18.4:
resolution: {integrity: sha512-1NxP+iOk8KSvS1L9SSxEvBAJk39U0GiGZkiiJGbuDF9G4fG7DSDw6XLxZMecAgmvQrwwx7yVKdNN3GgNh0UfKg==}
/@esbuild/win32-arm64@0.18.10:
resolution: {integrity: sha512-kRhNcMZFGMW+ZHCarAM1ypr8OZs0k688ViUCetVCef9p3enFxzWeBg9h/575Y0nsFu0ZItluCVF5gMR2pwOEpA==}
engines: {node: '>=12'}
cpu: [arm64]
os: [win32]
@ -712,8 +717,8 @@ packages:
dev: true
optional: true
/@esbuild/win32-ia32@0.18.4:
resolution: {integrity: sha512-OKr8jze93vbgqZ/r23woWciTixUwLa976C9W7yNBujtnVHyvsL/ocYG61tsktUfJOpyIz5TsohkBZ6Lo2+PCcQ==}
/@esbuild/win32-ia32@0.18.10:
resolution: {integrity: sha512-AR9PX1whYaYh9p0EOaKna0h48F/A101Mt/ag72+kMkkBZXPQ7cjbz2syXI/HI3OlBdUytSdHneljfjvUoqwqiQ==}
engines: {node: '>=12'}
cpu: [ia32]
os: [win32]
@ -730,8 +735,8 @@ packages:
dev: true
optional: true
/@esbuild/win32-x64@0.18.4:
resolution: {integrity: sha512-qJr3wVvcLjPFcV4AMDS3iquhBfTef2zo/jlm8RMxmiRp3Vy2HY8WMxrykJlcbCnqLXZPA0YZxZGND6eug85ogg==}
/@esbuild/win32-x64@0.18.10:
resolution: {integrity: sha512-5sTkYhAGHNRr6bVf4RM0PsscqVr6/DBYdrlMh168oph3usid3lKHcHEEHmr34iZ9GHeeg2juFOxtpl6XyC3tpw==}
engines: {node: '>=12'}
cpu: [x64]
os: [win32]
@ -1052,7 +1057,7 @@ packages:
resolution: {integrity: sha512-9GcLXF0/v3t80caGs5p2rRfkB+a8VBGLJZVih6CNFkx8IZ994wiKKLSRs9nuFwk1HevWs/1mnUmkApGrSGsShA==}
dev: true
/@typescript-eslint/eslint-plugin@5.60.1(@typescript-eslint/parser@5.60.1)(eslint@8.43.0)(typescript@5.1.3):
/@typescript-eslint/eslint-plugin@5.60.1(@typescript-eslint/parser@5.60.1)(eslint@8.43.0)(typescript@5.1.6):
resolution: {integrity: sha512-KSWsVvsJsLJv3c4e73y/Bzt7OpqMCADUO846bHcuWYSYM19bldbAeDv7dYyV0jwkbMfJ2XdlzwjhXtuD7OY6bw==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
peerDependencies:
@ -1064,23 +1069,23 @@ packages:
optional: true
dependencies:
'@eslint-community/regexpp': 4.5.1
'@typescript-eslint/parser': 5.60.1(eslint@8.43.0)(typescript@5.1.3)
'@typescript-eslint/parser': 5.60.1(eslint@8.43.0)(typescript@5.1.6)
'@typescript-eslint/scope-manager': 5.60.1
'@typescript-eslint/type-utils': 5.60.1(eslint@8.43.0)(typescript@5.1.3)
'@typescript-eslint/utils': 5.60.1(eslint@8.43.0)(typescript@5.1.3)
'@typescript-eslint/type-utils': 5.60.1(eslint@8.43.0)(typescript@5.1.6)
'@typescript-eslint/utils': 5.60.1(eslint@8.43.0)(typescript@5.1.6)
debug: 4.3.4
eslint: 8.43.0
grapheme-splitter: 1.0.4
ignore: 5.2.4
natural-compare-lite: 1.4.0
semver: 7.5.2
tsutils: 3.21.0(typescript@5.1.3)
typescript: 5.1.3
semver: 7.5.3
tsutils: 3.21.0(typescript@5.1.6)
typescript: 5.1.6
transitivePeerDependencies:
- supports-color
dev: true
/@typescript-eslint/parser@5.60.1(eslint@8.43.0)(typescript@5.1.3):
/@typescript-eslint/parser@5.60.1(eslint@8.43.0)(typescript@5.1.6):
resolution: {integrity: sha512-pHWlc3alg2oSMGwsU/Is8hbm3XFbcrb6P5wIxcQW9NsYBfnrubl/GhVVD/Jm/t8HXhA2WncoIRfBtnCgRGV96Q==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
peerDependencies:
@ -1092,10 +1097,10 @@ packages:
dependencies:
'@typescript-eslint/scope-manager': 5.60.1
'@typescript-eslint/types': 5.60.1
'@typescript-eslint/typescript-estree': 5.60.1(typescript@5.1.3)
'@typescript-eslint/typescript-estree': 5.60.1(typescript@5.1.6)
debug: 4.3.4
eslint: 8.43.0
typescript: 5.1.3
typescript: 5.1.6
transitivePeerDependencies:
- supports-color
dev: true
@ -1108,7 +1113,7 @@ packages:
'@typescript-eslint/visitor-keys': 5.60.1
dev: true
/@typescript-eslint/type-utils@5.60.1(eslint@8.43.0)(typescript@5.1.3):
/@typescript-eslint/type-utils@5.60.1(eslint@8.43.0)(typescript@5.1.6):
resolution: {integrity: sha512-vN6UztYqIu05nu7JqwQGzQKUJctzs3/Hg7E2Yx8rz9J+4LgtIDFWjjl1gm3pycH0P3mHAcEUBd23LVgfrsTR8A==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
peerDependencies:
@ -1118,12 +1123,12 @@ packages:
typescript:
optional: true
dependencies:
'@typescript-eslint/typescript-estree': 5.60.1(typescript@5.1.3)
'@typescript-eslint/utils': 5.60.1(eslint@8.43.0)(typescript@5.1.3)
'@typescript-eslint/typescript-estree': 5.60.1(typescript@5.1.6)
'@typescript-eslint/utils': 5.60.1(eslint@8.43.0)(typescript@5.1.6)
debug: 4.3.4
eslint: 8.43.0
tsutils: 3.21.0(typescript@5.1.3)
typescript: 5.1.3
tsutils: 3.21.0(typescript@5.1.6)
typescript: 5.1.6
transitivePeerDependencies:
- supports-color
dev: true
@ -1133,7 +1138,7 @@ packages:
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
dev: true
/@typescript-eslint/typescript-estree@5.60.1(typescript@5.1.3):
/@typescript-eslint/typescript-estree@5.60.1(typescript@5.1.6):
resolution: {integrity: sha512-hkX70J9+2M2ZT6fhti5Q2FoU9zb+GeZK2SLP1WZlvUDqdMbEKhexZODD1WodNRyO8eS+4nScvT0dts8IdaBzfw==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
peerDependencies:
@ -1147,14 +1152,14 @@ packages:
debug: 4.3.4
globby: 11.1.0
is-glob: 4.0.3
semver: 7.5.2
tsutils: 3.21.0(typescript@5.1.3)
typescript: 5.1.3
semver: 7.5.3
tsutils: 3.21.0(typescript@5.1.6)
typescript: 5.1.6
transitivePeerDependencies:
- supports-color
dev: true
/@typescript-eslint/utils@5.60.1(eslint@8.43.0)(typescript@5.1.3):
/@typescript-eslint/utils@5.60.1(eslint@8.43.0)(typescript@5.1.6):
resolution: {integrity: sha512-tiJ7FFdFQOWssFa3gqb94Ilexyw0JVxj6vBzaSpfN/8IhoKkDuSAenUKvsSHw2A/TMpJb26izIszTXaqygkvpQ==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
peerDependencies:
@ -1165,10 +1170,10 @@ packages:
'@types/semver': 7.5.0
'@typescript-eslint/scope-manager': 5.60.1
'@typescript-eslint/types': 5.60.1
'@typescript-eslint/typescript-estree': 5.60.1(typescript@5.1.3)
'@typescript-eslint/typescript-estree': 5.60.1(typescript@5.1.6)
eslint: 8.43.0
eslint-scope: 5.1.1
semver: 7.5.2
semver: 7.5.3
transitivePeerDependencies:
- supports-color
- typescript
@ -1353,7 +1358,7 @@ packages:
debug: 4.3.4
emittery: 1.0.1
figures: 5.0.0
globby: 13.1.4
globby: 13.2.0
ignore-by-default: 2.1.0
indent-string: 5.0.0
is-error: 2.2.2
@ -1425,13 +1430,13 @@ packages:
resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}
dev: true
/bundle-require@4.0.1(esbuild@0.18.4):
/bundle-require@4.0.1(esbuild@0.18.10):
resolution: {integrity: sha512-9NQkRHlNdNpDBGmLpngF3EFDcwodhMUuLz9PaWYciVcQF9SE4LFjM2DB/xV1Li5JiuDMv7ZUWuC3rGbqR0MAXQ==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
peerDependencies:
esbuild: '>=0.17'
dependencies:
esbuild: 0.18.4
esbuild: 0.18.10
load-tsconfig: 0.2.5
dev: true
@ -1680,7 +1685,7 @@ packages:
js-string-escape: 1.0.1
lodash: 4.17.21
md5-hex: 3.0.1
semver: 7.5.2
semver: 7.5.3
well-known-symbols: 2.0.0
dev: true
@ -1793,7 +1798,7 @@ packages:
resolution: {integrity: sha512-tQbV/4u5WVB8HMJr08pgw0b6nG4RGt/tj+7Numvq+zqcvUFeMaIWWOUFltiU+6go8BSO2/ogsB4EasDaj0y68Q==}
engines: {node: '>=14.16'}
dependencies:
globby: 13.1.4
globby: 13.2.0
graceful-fs: 4.2.11
is-glob: 4.0.3
is-path-cwd: 3.0.0
@ -1942,34 +1947,34 @@ packages:
'@esbuild/win32-x64': 0.17.19
dev: true
/esbuild@0.18.4:
resolution: {integrity: sha512-9rxWV/Cb2DMUXfe9aUsYtqg0KTlw146ElFH22kYeK9KVV1qT082X4lpmiKsa12ePiCcIcB686TQJxaGAa9TFvA==}
/esbuild@0.18.10:
resolution: {integrity: sha512-33WKo67auOXzZHBY/9DTJRo7kIvfU12S+D4sp2wIz39N88MDIaCGyCwbW01RR70pK6Iya0I74lHEpyLfFqOHPA==}
engines: {node: '>=12'}
hasBin: true
requiresBuild: true
optionalDependencies:
'@esbuild/android-arm': 0.18.4
'@esbuild/android-arm64': 0.18.4
'@esbuild/android-x64': 0.18.4
'@esbuild/darwin-arm64': 0.18.4
'@esbuild/darwin-x64': 0.18.4
'@esbuild/freebsd-arm64': 0.18.4
'@esbuild/freebsd-x64': 0.18.4
'@esbuild/linux-arm': 0.18.4
'@esbuild/linux-arm64': 0.18.4
'@esbuild/linux-ia32': 0.18.4
'@esbuild/linux-loong64': 0.18.4
'@esbuild/linux-mips64el': 0.18.4
'@esbuild/linux-ppc64': 0.18.4
'@esbuild/linux-riscv64': 0.18.4
'@esbuild/linux-s390x': 0.18.4
'@esbuild/linux-x64': 0.18.4
'@esbuild/netbsd-x64': 0.18.4
'@esbuild/openbsd-x64': 0.18.4
'@esbuild/sunos-x64': 0.18.4
'@esbuild/win32-arm64': 0.18.4
'@esbuild/win32-ia32': 0.18.4
'@esbuild/win32-x64': 0.18.4
'@esbuild/android-arm': 0.18.10
'@esbuild/android-arm64': 0.18.10
'@esbuild/android-x64': 0.18.10
'@esbuild/darwin-arm64': 0.18.10
'@esbuild/darwin-x64': 0.18.10
'@esbuild/freebsd-arm64': 0.18.10
'@esbuild/freebsd-x64': 0.18.10
'@esbuild/linux-arm': 0.18.10
'@esbuild/linux-arm64': 0.18.10
'@esbuild/linux-ia32': 0.18.10
'@esbuild/linux-loong64': 0.18.10
'@esbuild/linux-mips64el': 0.18.10
'@esbuild/linux-ppc64': 0.18.10
'@esbuild/linux-riscv64': 0.18.10
'@esbuild/linux-s390x': 0.18.10
'@esbuild/linux-x64': 0.18.10
'@esbuild/netbsd-x64': 0.18.10
'@esbuild/openbsd-x64': 0.18.10
'@esbuild/sunos-x64': 0.18.10
'@esbuild/win32-arm64': 0.18.10
'@esbuild/win32-ia32': 0.18.10
'@esbuild/win32-x64': 0.18.10
dev: true
/escalade@3.1.1:
@ -2064,7 +2069,7 @@ packages:
lodash.merge: 4.6.2
minimatch: 3.1.2
natural-compare: 1.4.0
optionator: 0.9.1
optionator: 0.9.3
strip-ansi: 6.0.1
strip-json-comments: 3.1.1
text-table: 0.2.0
@ -2325,8 +2330,8 @@ packages:
get-intrinsic: 1.2.1
dev: true
/get-tsconfig@4.6.0:
resolution: {integrity: sha512-lgbo68hHTQnFddybKbbs/RDRJnJT5YyGy2kQzVwbq+g67X73i+5MVTval34QxGkOe9X5Ujf1UYpCaphLyltjEg==}
/get-tsconfig@4.6.2:
resolution: {integrity: sha512-E5XrT4CbbXcXWy+1jChlZmrmCwd5KGx502kDCXJJ7y898TtWW9FwoG5HfOLVRKmlmDGkWN2HM9Ho+/Y8F0sJDg==}
dependencies:
resolve-pkg-maps: 1.0.0
dev: true
@ -2398,8 +2403,8 @@ packages:
slash: 3.0.0
dev: true
/globby@13.1.4:
resolution: {integrity: sha512-iui/IiiW+QrJ1X1hKH5qwlMQyv34wJAYwH1vrf8b9kBA4sNiif3gKsMHa+BrdnOpEudWjpotfa7LrTzB1ERS/g==}
/globby@13.2.0:
resolution: {integrity: sha512-jWsQfayf13NvqKUIL3Ta+CIqMnvlaIDFveWE/dpOZ9+3AMEJozsxDvKA02zync9UuvOM8rOXzsD5GqKP4OnWPQ==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
dependencies:
dir-glob: 3.0.1
@ -2958,8 +2963,8 @@ packages:
resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==}
dev: true
/lint-staged@13.2.2:
resolution: {integrity: sha512-71gSwXKy649VrSU09s10uAT0rWCcY3aewhMaHyl2N84oBk4Xs9HgxvUp3AYu+bNsK4NrOYYxvSgg7FyGJ+jGcA==}
/lint-staged@13.2.3:
resolution: {integrity: sha512-zVVEXLuQIhr1Y7R7YAWx4TZLdvuzk7DnmrsTNL0fax6Z3jrpFcas+vKbzxhhvp6TA55m1SQuWkpzI1qbfDZbAg==}
engines: {node: ^14.13.1 || >=16.0.0}
hasBin: true
dependencies:
@ -3303,7 +3308,7 @@ packages:
dependencies:
hosted-git-info: 4.1.0
is-core-module: 2.12.1
semver: 7.5.2
semver: 7.5.3
validate-npm-package-license: 3.0.4
dev: true
@ -3398,16 +3403,16 @@ packages:
zod: 3.21.4
dev: false
/optionator@0.9.1:
resolution: {integrity: sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==}
/optionator@0.9.3:
resolution: {integrity: sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==}
engines: {node: '>= 0.8.0'}
dependencies:
'@aashutoshrathi/word-wrap': 1.2.6
deep-is: 0.1.4
fast-levenshtein: 2.0.6
levn: 0.4.1
prelude-ls: 1.2.1
type-check: 0.4.0
word-wrap: 1.2.3
dev: true
/os-tmpdir@1.0.2:
@ -3609,8 +3614,8 @@ packages:
engines: {node: '>=4'}
dev: true
/pirates@4.0.5:
resolution: {integrity: sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ==}
/pirates@4.0.6:
resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==}
engines: {node: '>= 6'}
dev: true
@ -3829,8 +3834,8 @@ packages:
glob: 7.2.3
dev: true
/rollup@3.25.1:
resolution: {integrity: sha512-tywOR+rwIt5m2ZAWSe5AIJcTat8vGlnPFAv15ycCrw33t6iFsXZ6mzHVFh2psSjxQPmI+xgzMZZizUAukBI4aQ==}
/rollup@3.25.3:
resolution: {integrity: sha512-ZT279hx8gszBj9uy5FfhoG4bZx8c+0A1sbqtr7Q3KNWIizpTdDEPZbV2xcbvHsnFp4MavCQYZyzApJ+virB8Yw==}
engines: {node: '>=14.18.0', npm: '>=8.0.0'}
hasBin: true
optionalDependencies:
@ -3851,7 +3856,7 @@ packages:
/rxjs@7.8.1:
resolution: {integrity: sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==}
dependencies:
tslib: 2.5.3
tslib: 2.6.0
dev: true
/safe-regex-test@1.0.0:
@ -3876,8 +3881,8 @@ packages:
hasBin: true
dev: true
/semver@7.5.2:
resolution: {integrity: sha512-SoftuTROv/cRjCze/scjGyiDtcUyxw1rgYQSZY7XTmtR5hX+dm76iDbTH8TkLPHCQmlbQVSSbNZCPM2hb0knnQ==}
/semver@7.5.3:
resolution: {integrity: sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==}
engines: {node: '>=10'}
hasBin: true
dependencies:
@ -4165,7 +4170,7 @@ packages:
glob: 7.1.6
lines-and-columns: 1.2.4
mz: 2.7.0
pirates: 4.0.5
pirates: 4.0.6
ts-interface-checker: 0.1.13
dev: true
@ -4289,11 +4294,11 @@ packages:
resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==}
dev: true
/tslib@2.5.3:
resolution: {integrity: sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==}
/tslib@2.6.0:
resolution: {integrity: sha512-7At1WUettjcSRHXCyYtTselblcHl9PJFFVKiCAy/bY97+BPZXSQ2wbq0P9s8tK2G7dFQfNnlJnPAiArVBVBsfA==}
dev: true
/tsup@7.1.0(typescript@5.1.3):
/tsup@7.1.0(typescript@5.1.6):
resolution: {integrity: sha512-mazl/GRAk70j8S43/AbSYXGgvRP54oQeX8Un4iZxzATHt0roW0t6HYDVZIXMw0ZQIpvr1nFMniIVnN5186lW7w==}
engines: {node: '>=16.14'}
hasBin: true
@ -4309,34 +4314,34 @@ packages:
typescript:
optional: true
dependencies:
bundle-require: 4.0.1(esbuild@0.18.4)
bundle-require: 4.0.1(esbuild@0.18.10)
cac: 6.7.14
chokidar: 3.5.3
debug: 4.3.4
esbuild: 0.18.4
esbuild: 0.18.10
execa: 5.1.1
globby: 11.1.0
joycon: 3.1.1
postcss-load-config: 4.0.1
resolve-from: 5.0.0
rollup: 3.25.1
rollup: 3.25.3
source-map: 0.8.0-beta.0
sucrase: 3.32.0
tree-kill: 1.2.2
typescript: 5.1.3
typescript: 5.1.6
transitivePeerDependencies:
- supports-color
- ts-node
dev: true
/tsutils@3.21.0(typescript@5.1.3):
/tsutils@3.21.0(typescript@5.1.6):
resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==}
engines: {node: '>= 6'}
peerDependencies:
typescript: '>=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta'
dependencies:
tslib: 1.14.1
typescript: 5.1.3
typescript: 5.1.6
dev: true
/tsx@3.12.7:
@ -4401,10 +4406,10 @@ packages:
typedoc: '>=0.24.0'
dependencies:
prettier: 2.8.8
typedoc: 0.24.8(typescript@5.1.3)
typedoc: 0.24.8(typescript@5.1.6)
dev: true
/typedoc@0.24.8(typescript@5.1.3):
/typedoc@0.24.8(typescript@5.1.6):
resolution: {integrity: sha512-ahJ6Cpcvxwaxfu4KtjA8qZNqS43wYt6JL27wYiIgl1vd38WW/KWX11YuAeZhuz9v+ttrutSsgK+XO1CjL1kA3w==}
engines: {node: '>= 14.14'}
hasBin: true
@ -4415,11 +4420,11 @@ packages:
marked: 4.3.0
minimatch: 9.0.2
shiki: 0.14.3
typescript: 5.1.3
typescript: 5.1.6
dev: true
/typescript@5.1.3:
resolution: {integrity: sha512-XH627E9vkeqhlZFQuL+UsyAXEnibT0kWR2FWONlr4sTjvxyJYnyefgrkyECLzM5NenmKzRAy2rR/OlYLA1HkZw==}
/typescript@5.1.6:
resolution: {integrity: sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==}
engines: {node: '>=14.17'}
hasBin: true
dev: true
@ -4536,11 +4541,6 @@ packages:
isexe: 2.0.0
dev: true
/word-wrap@1.2.3:
resolution: {integrity: sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==}
engines: {node: '>=0.10.0'}
dev: true
/wordwrap@1.0.0:
resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==}
dev: false
@ -4634,8 +4634,8 @@ packages:
engines: {node: '>=12.20'}
dev: true
/zod-to-json-schema@3.21.2(zod@3.21.4):
resolution: {integrity: sha512-02yfKymfmIf2rM/5LYGlyw0daEel/f3MsSGMNJZWWf44ato+Y+diFugOpDtgvEUn3cYM5oDAGWW2NHeSD4mByw==}
/zod-to-json-schema@3.21.3(zod@3.21.4):
resolution: {integrity: sha512-09W/9oyxeF1/wWnzCb6MursW+lOzgKi91QwE7eTBbC+t/qgfuLsUVDai3lHemSQnQu/UONAcT/fv3ZnDvbTeKg==}
peerDependencies:
zod: ^3.21.4
dependencies:

34
legacy/src/tokenizer.ts
Wyświetl plik

@ -1,12 +1,10 @@
import {
Tiktoken,
TiktokenBPE,
TiktokenEncoding,
TiktokenModel,
getEncoding,
getEncodingNameForModel
} from 'js-tiktoken/lite'
import ky from 'ky'
import pMemoize from 'p-memoize'
} from 'js-tiktoken'
export interface Tokenizer {
encode(
@ -46,31 +44,6 @@ export class TiktokenTokenizer implements Tokenizer {
}
}
export const getTiktokenBPE = pMemoize(getTiktokenBPEImpl)
/**
* Asynchronously retrieves the Byte Pair Encoding (BPE) for a specified Tiktoken encoding.
*
* @param encoding - Tiktoken encoding
* @param options - optional settings for the request
* @returns promise that resolves to the BPE for the specified encoding
*/
async function getTiktokenBPEImpl(
encoding: TiktokenEncoding,
{
signal,
timeoutMs = 30000
}: {
signal?: AbortSignal
timeoutMs?: number
} = {}
) {
return ky(`https://tiktoken.pages.dev/js/${encoding}.json`, {
signal,
timeout: timeoutMs
}).json<TiktokenBPE>()
}
/**
* Asynchronously creates and retrieves a tokenizer for a specified Tiktoken encoding.
*
@ -86,8 +59,7 @@ export async function getTokenizerForEncoding(
extendedSpecialTokens?: Record<string, number>
}
) {
const tiktokenBPE = await getTiktokenBPE(encoding, options)
const tiktoken = new Tiktoken(tiktokenBPE, options?.extendedSpecialTokens)
const tiktoken = getEncoding(encoding, options?.extendedSpecialTokens)
return new TiktokenTokenizer(tiktoken)
}

12
legacy/test/tokenizer.test.ts vendored
Wyświetl plik

@ -7,13 +7,21 @@ import './_utils'
const models = [
'gpt-3.5-turbo',
'gpt-4',
'gpt-4',
'gpt-4',
'gpt-4',
'gpt-4',
'gpt-4',
'gpt-4-0613',
'text-davinci-003',
'code-davinci-002'
]
for (const model of models) {
test(`getTokenizerForModel ${model}`, async (t) => {
for (let i = 0; i < models.length; ++i) {
const model = models[i]
test(`getTokenizerForModel ${model} (${i})`, async (t) => {
t.timeout(10_000)
const tokenizer = await tokenizers.getTokenizerForModel(model)
t.truthy(tokenizer)