kopia lustrzana https://github.com/transitive-bullshit/chatgpt-api
feat: switch tokenizer to use cl100k_base encoding for gpt-3.5-turbo model
rodzic
ff269eb4fe
commit
95176fa0f2
|
@ -1,7 +1,7 @@
|
||||||
import { encoding_for_model } from '@dqbd/tiktoken'
|
import { get_encoding } from '@dqbd/tiktoken'
|
||||||
|
|
||||||
// TODO: make this configurable
|
// TODO: make this configurable
|
||||||
const tokenizer = encoding_for_model('text-davinci-003')
|
const tokenizer = get_encoding('cl100k_base')
|
||||||
|
|
||||||
export function encode(input: string): Uint32Array {
|
export function encode(input: string): Uint32Array {
|
||||||
return tokenizer.encode(input)
|
return tokenizer.encode(input)
|
||||||
|
|
Ładowanie…
Reference in New Issue