kopia lustrzana https://github.com/transitive-bullshit/chatgpt-api
feat: switch tokenizer to use cl100k_base encoding for gpt-3.5-turbo model
rodzic
e7f24b8c3d
commit
a733b29861
|
@ -1,7 +1,7 @@
|
|||
import { encoding_for_model } from '@dqbd/tiktoken'
|
||||
import { get_encoding } from '@dqbd/tiktoken'
|
||||
|
||||
// TODO: make this configurable
|
||||
const tokenizer = encoding_for_model('text-davinci-003')
|
||||
const tokenizer = get_encoding('cl100k_base')
|
||||
|
||||
export function encode(input: string): Uint32Array {
|
||||
return tokenizer.encode(input)
|
||||
|
|
Ładowanie…
Reference in New Issue