From 18e015029c0de8ef3b84f8be78ac535891df470c Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Wed, 28 Jun 2023 20:16:56 -0700 Subject: [PATCH] feat: switch js-tiktoken to load tokenizer ranks eagerly --- legacy/package.json | 6 +- legacy/pnpm-lock.yaml | 294 +++++++++++++++++----------------- legacy/src/tokenizer.ts | 34 +--- legacy/test/tokenizer.test.ts | 12 +- 4 files changed, 163 insertions(+), 183 deletions(-) diff --git a/legacy/package.json b/legacy/package.json index a8045a6d..764df455 100644 --- a/legacy/package.json +++ b/legacy/package.json @@ -66,7 +66,7 @@ "replicate": "^0.12.3", "ts-dedent": "^2.2.0", "zod": "^3.21.4", - "zod-to-json-schema": "^3.21.2", + "zod-to-json-schema": "^3.21.3", "zod-validation-error": "^1.3.1" }, "devDependencies": { @@ -88,7 +88,7 @@ "husky": "^8.0.3", "ioredis": "^5.3.2", "keyv": "^4.5.2", - "lint-staged": "^13.2.2", + "lint-staged": "^13.2.3", "npm-run-all": "^4.1.5", "p-memoize": "^7.1.1", "prettier": "^2.8.8", @@ -99,7 +99,7 @@ "type-fest": "^3.12.0", "typedoc": "^0.24.8", "typedoc-plugin-markdown": "4.0.0-next.16", - "typescript": "^5.1.3" + "typescript": "^5.1.6" }, "lint-staged": { "*.{ts,tsx}": [ diff --git a/legacy/pnpm-lock.yaml b/legacy/pnpm-lock.yaml index 9e4534b8..fcf9c106 100644 --- a/legacy/pnpm-lock.yaml +++ b/legacy/pnpm-lock.yaml @@ -87,8 +87,8 @@ dependencies: specifier: ^3.21.4 version: 3.21.4 zod-to-json-schema: - specifier: ^3.21.2 - version: 3.21.2(zod@3.21.4) + specifier: ^3.21.3 + version: 3.21.3(zod@3.21.4) zod-validation-error: specifier: ^1.3.1 version: 1.3.1(zod@3.21.4) @@ -111,10 +111,10 @@ devDependencies: version: 10.0.15 '@typescript-eslint/eslint-plugin': specifier: ^5.60.1 - version: 5.60.1(@typescript-eslint/parser@5.60.1)(eslint@8.43.0)(typescript@5.1.3) + version: 5.60.1(@typescript-eslint/parser@5.60.1)(eslint@8.43.0)(typescript@5.1.6) '@typescript-eslint/parser': specifier: ^5.60.1 - version: 5.60.1(eslint@8.43.0)(typescript@5.1.3) + version: 5.60.1(eslint@8.43.0)(typescript@5.1.6) ava: specifier: ^5.3.1 version: 5.3.1 @@ -149,8 +149,8 @@ devDependencies: specifier: ^4.5.2 version: 4.5.2 lint-staged: - specifier: ^13.2.2 - version: 13.2.2 + specifier: ^13.2.3 + version: 13.2.3 npm-run-all: specifier: ^4.1.5 version: 4.1.5 @@ -168,7 +168,7 @@ devDependencies: version: 15.2.0 tsup: specifier: 7.1.0 - version: 7.1.0(typescript@5.1.3) + version: 7.1.0(typescript@5.1.6) tsx: specifier: ^3.12.7 version: 3.12.7 @@ -177,16 +177,21 @@ devDependencies: version: 3.12.0 typedoc: specifier: ^0.24.8 - version: 0.24.8(typescript@5.1.3) + version: 0.24.8(typescript@5.1.6) typedoc-plugin-markdown: specifier: 4.0.0-next.16 version: 4.0.0-next.16(prettier@2.8.8)(typedoc@0.24.8) typescript: - specifier: ^5.1.3 - version: 5.1.3 + specifier: ^5.1.6 + version: 5.1.6 packages: + /@aashutoshrathi/word-wrap@1.2.6: + resolution: {integrity: sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==} + engines: {node: '>=0.10.0'} + dev: true + /@agentic/midjourney-fetch@1.0.1: resolution: {integrity: sha512-CUGNh3YJfbAPrOKcmqOOWuWQrkgVY+pWbCGhZkq74Jlc6oP8Jb+/K61V7Jhf/liYgYopbrpoDDue+ZKqijW/2A==} engines: {node: '>=18', pnpm: '>=8'} @@ -326,7 +331,7 @@ packages: resolution: {integrity: sha512-BDXFbYOJzT/NBEtp71cvsrGPwGAMGRB/349rwKuoxNSiKjPraNNnlK6MIIabViCjqZugu6j+xeMDlEkWdHHJSg==} dependencies: '@esbuild-kit/core-utils': 3.1.0 - get-tsconfig: 4.6.0 + get-tsconfig: 4.6.2 dev: true /@esbuild-kit/core-utils@3.1.0: @@ -340,7 +345,7 @@ packages: resolution: {integrity: sha512-Qwfvj/qoPbClxCRNuac1Du01r9gvNOT+pMYtJDapfB1eoGN1YlJ1BixLyL9WVENRx5RXgNLdfYdx/CuswlGhMw==} dependencies: '@esbuild-kit/core-utils': 3.1.0 - get-tsconfig: 4.6.0 + get-tsconfig: 4.6.2 dev: true /@esbuild/android-arm64@0.17.19: @@ -352,8 +357,8 @@ packages: dev: true optional: true - /@esbuild/android-arm64@0.18.4: - resolution: {integrity: sha512-yQVgO+V307hA2XhzELQ6F91CBGX7gSnlVGAj5YIqjQOxThDpM7fOcHT2YLJbE6gNdPtgRSafQrsK8rJ9xHCaZg==} + /@esbuild/android-arm64@0.18.10: + resolution: {integrity: sha512-ynm4naLbNbK0ajf9LUWtQB+6Vfg1Z/AplArqr4tGebC00Z6m9Y91OVIcjDa461wGcZwcaHYaZAab4yJxfhisTQ==} engines: {node: '>=12'} cpu: [arm64] os: [android] @@ -370,8 +375,8 @@ packages: dev: true optional: true - /@esbuild/android-arm@0.18.4: - resolution: {integrity: sha512-yKmQC9IiuvHdsNEbPHSprnMHg6OhL1cSeQZLzPpgzJBJ9ppEg9GAZN8MKj1TcmB4tZZUrq5xjK7KCmhwZP8iDA==} + /@esbuild/android-arm@0.18.10: + resolution: {integrity: sha512-3KClmVNd+Fku82uZJz5C4Rx8m1PPmWUFz5Zkw8jkpZPOmsq+EG1TTOtw1OXkHuX3WczOFQigrtf60B1ijKwNsg==} engines: {node: '>=12'} cpu: [arm] os: [android] @@ -388,8 +393,8 @@ packages: dev: true optional: true - /@esbuild/android-x64@0.18.4: - resolution: {integrity: sha512-yLKXMxQg6sk1ntftxQ5uwyVgG4/S2E7UoOCc5N4YZW7fdkfRiYEXqm7CMuIfY2Vs3FTrNyKmSfNevIuIvJnMww==} + /@esbuild/android-x64@0.18.10: + resolution: {integrity: sha512-vFfXj8P9Yfjh54yqUDEHKzqzYuEfPyAOl3z7R9hjkwt+NCvbn9VMxX+IILnAfdImRBfYVItgSUsqGKhJFnBwZw==} engines: {node: '>=12'} cpu: [x64] os: [android] @@ -406,8 +411,8 @@ packages: dev: true optional: true - /@esbuild/darwin-arm64@0.18.4: - resolution: {integrity: sha512-MVPEoZjZpk2xQ1zckZrb8eQuQib+QCzdmMs3YZAYEQPg+Rztk5pUxGyk8htZOC8Z38NMM29W+MqY9Sqo/sDGKw==} + /@esbuild/darwin-arm64@0.18.10: + resolution: {integrity: sha512-k2OJQ7ZxE6sVc91+MQeZH9gFeDAH2uIYALPAwTjTCvcPy9Dzrf7V7gFUQPYkn09zloWhQ+nvxWHia2x2ZLR0sQ==} engines: {node: '>=12'} cpu: [arm64] os: [darwin] @@ -424,8 +429,8 @@ packages: dev: true optional: true - /@esbuild/darwin-x64@0.18.4: - resolution: {integrity: sha512-uEsRtYRUDsz7i2tXg/t/SyF+5gU1cvi9B6B8i5ebJgtUUHJYWyIPIesmIOL4/+bywjxsDMA/XrNFMgMffLnh5A==} + /@esbuild/darwin-x64@0.18.10: + resolution: {integrity: sha512-tnz/mdZk1L1Z3WpGjin/L2bKTe8/AKZpI8fcCLtH+gq8WXWsCNJSxlesAObV4qbtTl6pG5vmqFXfWUQ5hV8PAQ==} engines: {node: '>=12'} cpu: [x64] os: [darwin] @@ -442,8 +447,8 @@ packages: dev: true optional: true - /@esbuild/freebsd-arm64@0.18.4: - resolution: {integrity: sha512-I8EOigqWnOHRin6Zp5Y1cfH3oT54bd7Sdz/VnpUNksbOtfp8IWRTH4pgkgO5jWaRQPjCpJcOpdRjYAMjPt8wXg==} + /@esbuild/freebsd-arm64@0.18.10: + resolution: {integrity: sha512-QJluV0LwBrbHnYYwSKC+K8RGz0g/EyhpQH1IxdoFT0nM7PfgjE+aS8wxq/KFEsU0JkL7U/EEKd3O8xVBxXb2aA==} engines: {node: '>=12'} cpu: [arm64] os: [freebsd] @@ -460,8 +465,8 @@ packages: dev: true optional: true - /@esbuild/freebsd-x64@0.18.4: - resolution: {integrity: sha512-1bHfgMz/cNMjbpsYxjVgMJ1iwKq+NdDPlACBrWULD7ZdFmBQrhMicMaKb5CdmdVyvIwXmasOuF4r6Iq574kUTA==} + /@esbuild/freebsd-x64@0.18.10: + resolution: {integrity: sha512-Hi/ycUkS6KTw+U9G5PK5NoK7CZboicaKUSVs0FSiPNtuCTzK6HNM4DIgniH7hFaeuszDS9T4dhAHWiLSt/Y5Ng==} engines: {node: '>=12'} cpu: [x64] os: [freebsd] @@ -478,8 +483,8 @@ packages: dev: true optional: true - /@esbuild/linux-arm64@0.18.4: - resolution: {integrity: sha512-J42vLHaYREyiBwH0eQE4/7H1DTfZx8FuxyWSictx4d7ezzuKE3XOkIvOg+SQzRz7T9HLVKzq2tvbAov4UfufBw==} + /@esbuild/linux-arm64@0.18.10: + resolution: {integrity: sha512-Nz6XcfRBOO7jSrVpKAyEyFOPGhySPNlgumSDhWAspdQQ11ub/7/NZDMhWDFReE9QH/SsCOCLQbdj0atAk/HMOQ==} engines: {node: '>=12'} cpu: [arm64] os: [linux] @@ -496,8 +501,8 @@ packages: dev: true optional: true - /@esbuild/linux-arm@0.18.4: - resolution: {integrity: sha512-4XCGqM/Ay1LCXUBH59bL4JbSbbTK1K22dWHymWMGaEh2sQCDOUw+OQxozYV/YdBb91leK2NbuSrE2BRamwgaYw==} + /@esbuild/linux-arm@0.18.10: + resolution: {integrity: sha512-HfFoxY172tVHPIvJy+FHxzB4l8xU7e5cxmNS11cQ2jt4JWAukn/7LXaPdZid41UyTweqa4P/1zs201gRGCTwHw==} engines: {node: '>=12'} cpu: [arm] os: [linux] @@ -514,8 +519,8 @@ packages: dev: true optional: true - /@esbuild/linux-ia32@0.18.4: - resolution: {integrity: sha512-4ksIqFwhq7OExty7Sl1n0vqQSCqTG4sU6i99G2yuMr28CEOUZ/60N+IO9hwI8sIxBqmKmDgncE1n5CMu/3m0IA==} + /@esbuild/linux-ia32@0.18.10: + resolution: {integrity: sha512-otMdmSmkMe+pmiP/bZBjfphyAsTsngyT9RCYwoFzqrveAbux9nYitDTpdgToG0Z0U55+PnH654gCH2GQ1aB6Yw==} engines: {node: '>=12'} cpu: [ia32] os: [linux] @@ -532,8 +537,8 @@ packages: dev: true optional: true - /@esbuild/linux-loong64@0.18.4: - resolution: {integrity: sha512-bsWtoVHkGQgAsFXioDueXRiUIfSGrVkJjBBz4gcBJxXcD461cWFQFyu8Fxdj9TP+zEeqJ8C/O4LFFMBNi6Fscw==} + /@esbuild/linux-loong64@0.18.10: + resolution: {integrity: sha512-t8tjFuON1koxskzQ4VFoh0T5UDUMiLYjwf9Wktd0tx8AoK6xgU+5ubKOpWpcnhEQ2tESS5u0v6QuN8PX/ftwcQ==} engines: {node: '>=12'} cpu: [loong64] os: [linux] @@ -550,8 +555,8 @@ packages: dev: true optional: true - /@esbuild/linux-mips64el@0.18.4: - resolution: {integrity: sha512-LRD9Fu8wJQgIOOV1o3nRyzrheFYjxA0C1IVWZ93eNRRWBKgarYFejd5WBtrp43cE4y4D4t3qWWyklm73Mrsd/g==} + /@esbuild/linux-mips64el@0.18.10: + resolution: {integrity: sha512-+dUkcVzcfEJHz3HEnVpIJu8z8Wdn2n/nWMWdl6FVPFGJAVySO4g3+XPzNKFytVFwf8hPVDwYXzVcu8GMFqsqZw==} engines: {node: '>=12'} cpu: [mips64el] os: [linux] @@ -568,8 +573,8 @@ packages: dev: true optional: true - /@esbuild/linux-ppc64@0.18.4: - resolution: {integrity: sha512-jtQgoZjM92gauVRxNaaG/TpL3Pr4WcL3Pwqi9QgdrBGrEXzB+twohQiWNSTycs6lUygakos4mm2h0B9/SHveng==} + /@esbuild/linux-ppc64@0.18.10: + resolution: {integrity: sha512-sO3PjjxEGy+PY2qkGe2gwJbXdZN9wAYpVBZWFD0AwAoKuXRkWK0/zaMQ5ekUFJDRDCRm8x5U0Axaub7ynH/wVg==} engines: {node: '>=12'} cpu: [ppc64] os: [linux] @@ -586,8 +591,8 @@ packages: dev: true optional: true - /@esbuild/linux-riscv64@0.18.4: - resolution: {integrity: sha512-7WaU/kRZG0VCV09Xdlkg6LNAsfU9SAxo6XEdaZ8ffO4lh+DZoAhGTx7+vTMOXKxa+r2w1LYDGxfJa2rcgagMRA==} + /@esbuild/linux-riscv64@0.18.10: + resolution: {integrity: sha512-JDtdbJg3yjDeXLv4lZYE1kiTnxv73/8cbPHY9T/dUKi8rYOM/k5b3W4UJLMUksuQ6nTm5c89W1nADsql6FW75A==} engines: {node: '>=12'} cpu: [riscv64] os: [linux] @@ -604,8 +609,8 @@ packages: dev: true optional: true - /@esbuild/linux-s390x@0.18.4: - resolution: {integrity: sha512-D19ed0xreKQvC5t+ArE2njSnm18WPpE+1fhwaiJHf+Xwqsq+/SUaV8Mx0M27nszdU+Atq1HahrgCOZCNNEASUg==} + /@esbuild/linux-s390x@0.18.10: + resolution: {integrity: sha512-NLuSKcp8WckjD2a7z5kzLiCywFwBTMlIxDNuud1AUGVuwBBJSkuubp6cNjJ0p5c6CZaA3QqUGwjHJBiG1SoOFw==} engines: {node: '>=12'} cpu: [s390x] os: [linux] @@ -622,8 +627,8 @@ packages: dev: true optional: true - /@esbuild/linux-x64@0.18.4: - resolution: {integrity: sha512-Rx3AY1sxyiO/gvCGP00nL69L60dfmWyjKWY06ugpB8Ydpdsfi3BHW58HWC24K3CAjAPSwxcajozC2PzA9JBS1g==} + /@esbuild/linux-x64@0.18.10: + resolution: {integrity: sha512-wj2KRsCsFusli+6yFgNO/zmmLslislAWryJnodteRmGej7ZzinIbMdsyp13rVGde88zxJd5vercNYK9kuvlZaQ==} engines: {node: '>=12'} cpu: [x64] os: [linux] @@ -640,8 +645,8 @@ packages: dev: true optional: true - /@esbuild/netbsd-x64@0.18.4: - resolution: {integrity: sha512-AaShPmN9c6w1mKRpliKFlaWcSkpBT4KOlk93UfFgeI3F3cbjzdDKGsbKnOZozmYbE1izZKLmNJiW0sFM+A5JPA==} + /@esbuild/netbsd-x64@0.18.10: + resolution: {integrity: sha512-pQ9QqxEPI3cVRZyUtCoZxhZK3If+7RzR8L2yz2+TDzdygofIPOJFaAPkEJ5rYIbUO101RaiYxfdOBahYexLk5A==} engines: {node: '>=12'} cpu: [x64] os: [netbsd] @@ -658,8 +663,8 @@ packages: dev: true optional: true - /@esbuild/openbsd-x64@0.18.4: - resolution: {integrity: sha512-tRGvGwou3BrvHVvF8HxTqEiC5VtPzySudS9fh2jBIKpLX7HCW8jIkW+LunkFDNwhslx4xMAgh0jAHsx/iCymaQ==} + /@esbuild/openbsd-x64@0.18.10: + resolution: {integrity: sha512-k8GTIIW9I8pEEfoOUm32TpPMgSg06JhL5DO+ql66aLTkOQUs0TxCA67Wi7pv6z8iF8STCGcNbm3UWFHLuci+ag==} engines: {node: '>=12'} cpu: [x64] os: [openbsd] @@ -676,8 +681,8 @@ packages: dev: true optional: true - /@esbuild/sunos-x64@0.18.4: - resolution: {integrity: sha512-acORFDI95GKhmAnlH8EarBeuqoy/j3yxIU+FDB91H3+ZON+8HhTadtT450YkaMzX6lEWbhi+mjVUCj00M5yyOQ==} + /@esbuild/sunos-x64@0.18.10: + resolution: {integrity: sha512-vIGYJIdEI6d4JBucAx8py792G8J0GP40qSH+EvSt80A4zvGd6jph+5t1g+eEXcS2aRpgZw6CrssNCFZxTdEsxw==} engines: {node: '>=12'} cpu: [x64] os: [sunos] @@ -694,8 +699,8 @@ packages: dev: true optional: true - /@esbuild/win32-arm64@0.18.4: - resolution: {integrity: sha512-1NxP+iOk8KSvS1L9SSxEvBAJk39U0GiGZkiiJGbuDF9G4fG7DSDw6XLxZMecAgmvQrwwx7yVKdNN3GgNh0UfKg==} + /@esbuild/win32-arm64@0.18.10: + resolution: {integrity: sha512-kRhNcMZFGMW+ZHCarAM1ypr8OZs0k688ViUCetVCef9p3enFxzWeBg9h/575Y0nsFu0ZItluCVF5gMR2pwOEpA==} engines: {node: '>=12'} cpu: [arm64] os: [win32] @@ -712,8 +717,8 @@ packages: dev: true optional: true - /@esbuild/win32-ia32@0.18.4: - resolution: {integrity: sha512-OKr8jze93vbgqZ/r23woWciTixUwLa976C9W7yNBujtnVHyvsL/ocYG61tsktUfJOpyIz5TsohkBZ6Lo2+PCcQ==} + /@esbuild/win32-ia32@0.18.10: + resolution: {integrity: sha512-AR9PX1whYaYh9p0EOaKna0h48F/A101Mt/ag72+kMkkBZXPQ7cjbz2syXI/HI3OlBdUytSdHneljfjvUoqwqiQ==} engines: {node: '>=12'} cpu: [ia32] os: [win32] @@ -730,8 +735,8 @@ packages: dev: true optional: true - /@esbuild/win32-x64@0.18.4: - resolution: {integrity: sha512-qJr3wVvcLjPFcV4AMDS3iquhBfTef2zo/jlm8RMxmiRp3Vy2HY8WMxrykJlcbCnqLXZPA0YZxZGND6eug85ogg==} + /@esbuild/win32-x64@0.18.10: + resolution: {integrity: sha512-5sTkYhAGHNRr6bVf4RM0PsscqVr6/DBYdrlMh168oph3usid3lKHcHEEHmr34iZ9GHeeg2juFOxtpl6XyC3tpw==} engines: {node: '>=12'} cpu: [x64] os: [win32] @@ -1052,7 +1057,7 @@ packages: resolution: {integrity: sha512-9GcLXF0/v3t80caGs5p2rRfkB+a8VBGLJZVih6CNFkx8IZ994wiKKLSRs9nuFwk1HevWs/1mnUmkApGrSGsShA==} dev: true - /@typescript-eslint/eslint-plugin@5.60.1(@typescript-eslint/parser@5.60.1)(eslint@8.43.0)(typescript@5.1.3): + /@typescript-eslint/eslint-plugin@5.60.1(@typescript-eslint/parser@5.60.1)(eslint@8.43.0)(typescript@5.1.6): resolution: {integrity: sha512-KSWsVvsJsLJv3c4e73y/Bzt7OpqMCADUO846bHcuWYSYM19bldbAeDv7dYyV0jwkbMfJ2XdlzwjhXtuD7OY6bw==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: @@ -1064,23 +1069,23 @@ packages: optional: true dependencies: '@eslint-community/regexpp': 4.5.1 - '@typescript-eslint/parser': 5.60.1(eslint@8.43.0)(typescript@5.1.3) + '@typescript-eslint/parser': 5.60.1(eslint@8.43.0)(typescript@5.1.6) '@typescript-eslint/scope-manager': 5.60.1 - '@typescript-eslint/type-utils': 5.60.1(eslint@8.43.0)(typescript@5.1.3) - '@typescript-eslint/utils': 5.60.1(eslint@8.43.0)(typescript@5.1.3) + '@typescript-eslint/type-utils': 5.60.1(eslint@8.43.0)(typescript@5.1.6) + '@typescript-eslint/utils': 5.60.1(eslint@8.43.0)(typescript@5.1.6) debug: 4.3.4 eslint: 8.43.0 grapheme-splitter: 1.0.4 ignore: 5.2.4 natural-compare-lite: 1.4.0 - semver: 7.5.2 - tsutils: 3.21.0(typescript@5.1.3) - typescript: 5.1.3 + semver: 7.5.3 + tsutils: 3.21.0(typescript@5.1.6) + typescript: 5.1.6 transitivePeerDependencies: - supports-color dev: true - /@typescript-eslint/parser@5.60.1(eslint@8.43.0)(typescript@5.1.3): + /@typescript-eslint/parser@5.60.1(eslint@8.43.0)(typescript@5.1.6): resolution: {integrity: sha512-pHWlc3alg2oSMGwsU/Is8hbm3XFbcrb6P5wIxcQW9NsYBfnrubl/GhVVD/Jm/t8HXhA2WncoIRfBtnCgRGV96Q==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: @@ -1092,10 +1097,10 @@ packages: dependencies: '@typescript-eslint/scope-manager': 5.60.1 '@typescript-eslint/types': 5.60.1 - '@typescript-eslint/typescript-estree': 5.60.1(typescript@5.1.3) + '@typescript-eslint/typescript-estree': 5.60.1(typescript@5.1.6) debug: 4.3.4 eslint: 8.43.0 - typescript: 5.1.3 + typescript: 5.1.6 transitivePeerDependencies: - supports-color dev: true @@ -1108,7 +1113,7 @@ packages: '@typescript-eslint/visitor-keys': 5.60.1 dev: true - /@typescript-eslint/type-utils@5.60.1(eslint@8.43.0)(typescript@5.1.3): + /@typescript-eslint/type-utils@5.60.1(eslint@8.43.0)(typescript@5.1.6): resolution: {integrity: sha512-vN6UztYqIu05nu7JqwQGzQKUJctzs3/Hg7E2Yx8rz9J+4LgtIDFWjjl1gm3pycH0P3mHAcEUBd23LVgfrsTR8A==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: @@ -1118,12 +1123,12 @@ packages: typescript: optional: true dependencies: - '@typescript-eslint/typescript-estree': 5.60.1(typescript@5.1.3) - '@typescript-eslint/utils': 5.60.1(eslint@8.43.0)(typescript@5.1.3) + '@typescript-eslint/typescript-estree': 5.60.1(typescript@5.1.6) + '@typescript-eslint/utils': 5.60.1(eslint@8.43.0)(typescript@5.1.6) debug: 4.3.4 eslint: 8.43.0 - tsutils: 3.21.0(typescript@5.1.3) - typescript: 5.1.3 + tsutils: 3.21.0(typescript@5.1.6) + typescript: 5.1.6 transitivePeerDependencies: - supports-color dev: true @@ -1133,7 +1138,7 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dev: true - /@typescript-eslint/typescript-estree@5.60.1(typescript@5.1.3): + /@typescript-eslint/typescript-estree@5.60.1(typescript@5.1.6): resolution: {integrity: sha512-hkX70J9+2M2ZT6fhti5Q2FoU9zb+GeZK2SLP1WZlvUDqdMbEKhexZODD1WodNRyO8eS+4nScvT0dts8IdaBzfw==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: @@ -1147,14 +1152,14 @@ packages: debug: 4.3.4 globby: 11.1.0 is-glob: 4.0.3 - semver: 7.5.2 - tsutils: 3.21.0(typescript@5.1.3) - typescript: 5.1.3 + semver: 7.5.3 + tsutils: 3.21.0(typescript@5.1.6) + typescript: 5.1.6 transitivePeerDependencies: - supports-color dev: true - /@typescript-eslint/utils@5.60.1(eslint@8.43.0)(typescript@5.1.3): + /@typescript-eslint/utils@5.60.1(eslint@8.43.0)(typescript@5.1.6): resolution: {integrity: sha512-tiJ7FFdFQOWssFa3gqb94Ilexyw0JVxj6vBzaSpfN/8IhoKkDuSAenUKvsSHw2A/TMpJb26izIszTXaqygkvpQ==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: @@ -1165,10 +1170,10 @@ packages: '@types/semver': 7.5.0 '@typescript-eslint/scope-manager': 5.60.1 '@typescript-eslint/types': 5.60.1 - '@typescript-eslint/typescript-estree': 5.60.1(typescript@5.1.3) + '@typescript-eslint/typescript-estree': 5.60.1(typescript@5.1.6) eslint: 8.43.0 eslint-scope: 5.1.1 - semver: 7.5.2 + semver: 7.5.3 transitivePeerDependencies: - supports-color - typescript @@ -1353,7 +1358,7 @@ packages: debug: 4.3.4 emittery: 1.0.1 figures: 5.0.0 - globby: 13.1.4 + globby: 13.2.0 ignore-by-default: 2.1.0 indent-string: 5.0.0 is-error: 2.2.2 @@ -1425,13 +1430,13 @@ packages: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} dev: true - /bundle-require@4.0.1(esbuild@0.18.4): + /bundle-require@4.0.1(esbuild@0.18.10): resolution: {integrity: sha512-9NQkRHlNdNpDBGmLpngF3EFDcwodhMUuLz9PaWYciVcQF9SE4LFjM2DB/xV1Li5JiuDMv7ZUWuC3rGbqR0MAXQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} peerDependencies: esbuild: '>=0.17' dependencies: - esbuild: 0.18.4 + esbuild: 0.18.10 load-tsconfig: 0.2.5 dev: true @@ -1680,7 +1685,7 @@ packages: js-string-escape: 1.0.1 lodash: 4.17.21 md5-hex: 3.0.1 - semver: 7.5.2 + semver: 7.5.3 well-known-symbols: 2.0.0 dev: true @@ -1793,7 +1798,7 @@ packages: resolution: {integrity: sha512-tQbV/4u5WVB8HMJr08pgw0b6nG4RGt/tj+7Numvq+zqcvUFeMaIWWOUFltiU+6go8BSO2/ogsB4EasDaj0y68Q==} engines: {node: '>=14.16'} dependencies: - globby: 13.1.4 + globby: 13.2.0 graceful-fs: 4.2.11 is-glob: 4.0.3 is-path-cwd: 3.0.0 @@ -1942,34 +1947,34 @@ packages: '@esbuild/win32-x64': 0.17.19 dev: true - /esbuild@0.18.4: - resolution: {integrity: sha512-9rxWV/Cb2DMUXfe9aUsYtqg0KTlw146ElFH22kYeK9KVV1qT082X4lpmiKsa12ePiCcIcB686TQJxaGAa9TFvA==} + /esbuild@0.18.10: + resolution: {integrity: sha512-33WKo67auOXzZHBY/9DTJRo7kIvfU12S+D4sp2wIz39N88MDIaCGyCwbW01RR70pK6Iya0I74lHEpyLfFqOHPA==} engines: {node: '>=12'} hasBin: true requiresBuild: true optionalDependencies: - '@esbuild/android-arm': 0.18.4 - '@esbuild/android-arm64': 0.18.4 - '@esbuild/android-x64': 0.18.4 - '@esbuild/darwin-arm64': 0.18.4 - '@esbuild/darwin-x64': 0.18.4 - '@esbuild/freebsd-arm64': 0.18.4 - '@esbuild/freebsd-x64': 0.18.4 - '@esbuild/linux-arm': 0.18.4 - '@esbuild/linux-arm64': 0.18.4 - '@esbuild/linux-ia32': 0.18.4 - '@esbuild/linux-loong64': 0.18.4 - '@esbuild/linux-mips64el': 0.18.4 - '@esbuild/linux-ppc64': 0.18.4 - '@esbuild/linux-riscv64': 0.18.4 - '@esbuild/linux-s390x': 0.18.4 - '@esbuild/linux-x64': 0.18.4 - '@esbuild/netbsd-x64': 0.18.4 - '@esbuild/openbsd-x64': 0.18.4 - '@esbuild/sunos-x64': 0.18.4 - '@esbuild/win32-arm64': 0.18.4 - '@esbuild/win32-ia32': 0.18.4 - '@esbuild/win32-x64': 0.18.4 + '@esbuild/android-arm': 0.18.10 + '@esbuild/android-arm64': 0.18.10 + '@esbuild/android-x64': 0.18.10 + '@esbuild/darwin-arm64': 0.18.10 + '@esbuild/darwin-x64': 0.18.10 + '@esbuild/freebsd-arm64': 0.18.10 + '@esbuild/freebsd-x64': 0.18.10 + '@esbuild/linux-arm': 0.18.10 + '@esbuild/linux-arm64': 0.18.10 + '@esbuild/linux-ia32': 0.18.10 + '@esbuild/linux-loong64': 0.18.10 + '@esbuild/linux-mips64el': 0.18.10 + '@esbuild/linux-ppc64': 0.18.10 + '@esbuild/linux-riscv64': 0.18.10 + '@esbuild/linux-s390x': 0.18.10 + '@esbuild/linux-x64': 0.18.10 + '@esbuild/netbsd-x64': 0.18.10 + '@esbuild/openbsd-x64': 0.18.10 + '@esbuild/sunos-x64': 0.18.10 + '@esbuild/win32-arm64': 0.18.10 + '@esbuild/win32-ia32': 0.18.10 + '@esbuild/win32-x64': 0.18.10 dev: true /escalade@3.1.1: @@ -2064,7 +2069,7 @@ packages: lodash.merge: 4.6.2 minimatch: 3.1.2 natural-compare: 1.4.0 - optionator: 0.9.1 + optionator: 0.9.3 strip-ansi: 6.0.1 strip-json-comments: 3.1.1 text-table: 0.2.0 @@ -2325,8 +2330,8 @@ packages: get-intrinsic: 1.2.1 dev: true - /get-tsconfig@4.6.0: - resolution: {integrity: sha512-lgbo68hHTQnFddybKbbs/RDRJnJT5YyGy2kQzVwbq+g67X73i+5MVTval34QxGkOe9X5Ujf1UYpCaphLyltjEg==} + /get-tsconfig@4.6.2: + resolution: {integrity: sha512-E5XrT4CbbXcXWy+1jChlZmrmCwd5KGx502kDCXJJ7y898TtWW9FwoG5HfOLVRKmlmDGkWN2HM9Ho+/Y8F0sJDg==} dependencies: resolve-pkg-maps: 1.0.0 dev: true @@ -2398,8 +2403,8 @@ packages: slash: 3.0.0 dev: true - /globby@13.1.4: - resolution: {integrity: sha512-iui/IiiW+QrJ1X1hKH5qwlMQyv34wJAYwH1vrf8b9kBA4sNiif3gKsMHa+BrdnOpEudWjpotfa7LrTzB1ERS/g==} + /globby@13.2.0: + resolution: {integrity: sha512-jWsQfayf13NvqKUIL3Ta+CIqMnvlaIDFveWE/dpOZ9+3AMEJozsxDvKA02zync9UuvOM8rOXzsD5GqKP4OnWPQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dependencies: dir-glob: 3.0.1 @@ -2958,8 +2963,8 @@ packages: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} dev: true - /lint-staged@13.2.2: - resolution: {integrity: sha512-71gSwXKy649VrSU09s10uAT0rWCcY3aewhMaHyl2N84oBk4Xs9HgxvUp3AYu+bNsK4NrOYYxvSgg7FyGJ+jGcA==} + /lint-staged@13.2.3: + resolution: {integrity: sha512-zVVEXLuQIhr1Y7R7YAWx4TZLdvuzk7DnmrsTNL0fax6Z3jrpFcas+vKbzxhhvp6TA55m1SQuWkpzI1qbfDZbAg==} engines: {node: ^14.13.1 || >=16.0.0} hasBin: true dependencies: @@ -3303,7 +3308,7 @@ packages: dependencies: hosted-git-info: 4.1.0 is-core-module: 2.12.1 - semver: 7.5.2 + semver: 7.5.3 validate-npm-package-license: 3.0.4 dev: true @@ -3398,16 +3403,16 @@ packages: zod: 3.21.4 dev: false - /optionator@0.9.1: - resolution: {integrity: sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==} + /optionator@0.9.3: + resolution: {integrity: sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==} engines: {node: '>= 0.8.0'} dependencies: + '@aashutoshrathi/word-wrap': 1.2.6 deep-is: 0.1.4 fast-levenshtein: 2.0.6 levn: 0.4.1 prelude-ls: 1.2.1 type-check: 0.4.0 - word-wrap: 1.2.3 dev: true /os-tmpdir@1.0.2: @@ -3609,8 +3614,8 @@ packages: engines: {node: '>=4'} dev: true - /pirates@4.0.5: - resolution: {integrity: sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ==} + /pirates@4.0.6: + resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==} engines: {node: '>= 6'} dev: true @@ -3829,8 +3834,8 @@ packages: glob: 7.2.3 dev: true - /rollup@3.25.1: - resolution: {integrity: sha512-tywOR+rwIt5m2ZAWSe5AIJcTat8vGlnPFAv15ycCrw33t6iFsXZ6mzHVFh2psSjxQPmI+xgzMZZizUAukBI4aQ==} + /rollup@3.25.3: + resolution: {integrity: sha512-ZT279hx8gszBj9uy5FfhoG4bZx8c+0A1sbqtr7Q3KNWIizpTdDEPZbV2xcbvHsnFp4MavCQYZyzApJ+virB8Yw==} engines: {node: '>=14.18.0', npm: '>=8.0.0'} hasBin: true optionalDependencies: @@ -3851,7 +3856,7 @@ packages: /rxjs@7.8.1: resolution: {integrity: sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==} dependencies: - tslib: 2.5.3 + tslib: 2.6.0 dev: true /safe-regex-test@1.0.0: @@ -3876,8 +3881,8 @@ packages: hasBin: true dev: true - /semver@7.5.2: - resolution: {integrity: sha512-SoftuTROv/cRjCze/scjGyiDtcUyxw1rgYQSZY7XTmtR5hX+dm76iDbTH8TkLPHCQmlbQVSSbNZCPM2hb0knnQ==} + /semver@7.5.3: + resolution: {integrity: sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==} engines: {node: '>=10'} hasBin: true dependencies: @@ -4165,7 +4170,7 @@ packages: glob: 7.1.6 lines-and-columns: 1.2.4 mz: 2.7.0 - pirates: 4.0.5 + pirates: 4.0.6 ts-interface-checker: 0.1.13 dev: true @@ -4289,11 +4294,11 @@ packages: resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==} dev: true - /tslib@2.5.3: - resolution: {integrity: sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==} + /tslib@2.6.0: + resolution: {integrity: sha512-7At1WUettjcSRHXCyYtTselblcHl9PJFFVKiCAy/bY97+BPZXSQ2wbq0P9s8tK2G7dFQfNnlJnPAiArVBVBsfA==} dev: true - /tsup@7.1.0(typescript@5.1.3): + /tsup@7.1.0(typescript@5.1.6): resolution: {integrity: sha512-mazl/GRAk70j8S43/AbSYXGgvRP54oQeX8Un4iZxzATHt0roW0t6HYDVZIXMw0ZQIpvr1nFMniIVnN5186lW7w==} engines: {node: '>=16.14'} hasBin: true @@ -4309,34 +4314,34 @@ packages: typescript: optional: true dependencies: - bundle-require: 4.0.1(esbuild@0.18.4) + bundle-require: 4.0.1(esbuild@0.18.10) cac: 6.7.14 chokidar: 3.5.3 debug: 4.3.4 - esbuild: 0.18.4 + esbuild: 0.18.10 execa: 5.1.1 globby: 11.1.0 joycon: 3.1.1 postcss-load-config: 4.0.1 resolve-from: 5.0.0 - rollup: 3.25.1 + rollup: 3.25.3 source-map: 0.8.0-beta.0 sucrase: 3.32.0 tree-kill: 1.2.2 - typescript: 5.1.3 + typescript: 5.1.6 transitivePeerDependencies: - supports-color - ts-node dev: true - /tsutils@3.21.0(typescript@5.1.3): + /tsutils@3.21.0(typescript@5.1.6): resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} engines: {node: '>= 6'} peerDependencies: typescript: '>=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta' dependencies: tslib: 1.14.1 - typescript: 5.1.3 + typescript: 5.1.6 dev: true /tsx@3.12.7: @@ -4401,10 +4406,10 @@ packages: typedoc: '>=0.24.0' dependencies: prettier: 2.8.8 - typedoc: 0.24.8(typescript@5.1.3) + typedoc: 0.24.8(typescript@5.1.6) dev: true - /typedoc@0.24.8(typescript@5.1.3): + /typedoc@0.24.8(typescript@5.1.6): resolution: {integrity: sha512-ahJ6Cpcvxwaxfu4KtjA8qZNqS43wYt6JL27wYiIgl1vd38WW/KWX11YuAeZhuz9v+ttrutSsgK+XO1CjL1kA3w==} engines: {node: '>= 14.14'} hasBin: true @@ -4415,11 +4420,11 @@ packages: marked: 4.3.0 minimatch: 9.0.2 shiki: 0.14.3 - typescript: 5.1.3 + typescript: 5.1.6 dev: true - /typescript@5.1.3: - resolution: {integrity: sha512-XH627E9vkeqhlZFQuL+UsyAXEnibT0kWR2FWONlr4sTjvxyJYnyefgrkyECLzM5NenmKzRAy2rR/OlYLA1HkZw==} + /typescript@5.1.6: + resolution: {integrity: sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==} engines: {node: '>=14.17'} hasBin: true dev: true @@ -4536,11 +4541,6 @@ packages: isexe: 2.0.0 dev: true - /word-wrap@1.2.3: - resolution: {integrity: sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==} - engines: {node: '>=0.10.0'} - dev: true - /wordwrap@1.0.0: resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} dev: false @@ -4634,8 +4634,8 @@ packages: engines: {node: '>=12.20'} dev: true - /zod-to-json-schema@3.21.2(zod@3.21.4): - resolution: {integrity: sha512-02yfKymfmIf2rM/5LYGlyw0daEel/f3MsSGMNJZWWf44ato+Y+diFugOpDtgvEUn3cYM5oDAGWW2NHeSD4mByw==} + /zod-to-json-schema@3.21.3(zod@3.21.4): + resolution: {integrity: sha512-09W/9oyxeF1/wWnzCb6MursW+lOzgKi91QwE7eTBbC+t/qgfuLsUVDai3lHemSQnQu/UONAcT/fv3ZnDvbTeKg==} peerDependencies: zod: ^3.21.4 dependencies: diff --git a/legacy/src/tokenizer.ts b/legacy/src/tokenizer.ts index f87c595a..57cc1bd6 100644 --- a/legacy/src/tokenizer.ts +++ b/legacy/src/tokenizer.ts @@ -1,12 +1,10 @@ import { Tiktoken, - TiktokenBPE, TiktokenEncoding, TiktokenModel, + getEncoding, getEncodingNameForModel -} from 'js-tiktoken/lite' -import ky from 'ky' -import pMemoize from 'p-memoize' +} from 'js-tiktoken' export interface Tokenizer { encode( @@ -46,31 +44,6 @@ export class TiktokenTokenizer implements Tokenizer { } } -export const getTiktokenBPE = pMemoize(getTiktokenBPEImpl) - -/** - * Asynchronously retrieves the Byte Pair Encoding (BPE) for a specified Tiktoken encoding. - * - * @param encoding - Tiktoken encoding - * @param options - optional settings for the request - * @returns promise that resolves to the BPE for the specified encoding - */ -async function getTiktokenBPEImpl( - encoding: TiktokenEncoding, - { - signal, - timeoutMs = 30000 - }: { - signal?: AbortSignal - timeoutMs?: number - } = {} -) { - return ky(`https://tiktoken.pages.dev/js/${encoding}.json`, { - signal, - timeout: timeoutMs - }).json() -} - /** * Asynchronously creates and retrieves a tokenizer for a specified Tiktoken encoding. * @@ -86,8 +59,7 @@ export async function getTokenizerForEncoding( extendedSpecialTokens?: Record } ) { - const tiktokenBPE = await getTiktokenBPE(encoding, options) - const tiktoken = new Tiktoken(tiktokenBPE, options?.extendedSpecialTokens) + const tiktoken = getEncoding(encoding, options?.extendedSpecialTokens) return new TiktokenTokenizer(tiktoken) } diff --git a/legacy/test/tokenizer.test.ts b/legacy/test/tokenizer.test.ts index d65f62d7..804b242f 100644 --- a/legacy/test/tokenizer.test.ts +++ b/legacy/test/tokenizer.test.ts @@ -7,13 +7,21 @@ import './_utils' const models = [ 'gpt-3.5-turbo', 'gpt-4', + 'gpt-4', + 'gpt-4', + 'gpt-4', + 'gpt-4', + 'gpt-4', 'gpt-4-0613', 'text-davinci-003', 'code-davinci-002' ] -for (const model of models) { - test(`getTokenizerForModel ${model}`, async (t) => { +for (let i = 0; i < models.length; ++i) { + const model = models[i] + test(`getTokenizerForModel ${model} (${i})`, async (t) => { + t.timeout(10_000) + const tokenizer = await tokenizers.getTokenizerForModel(model) t.truthy(tokenizer)