docs: add TSDoc comments

src/llms/llm-utils.ts

@@ -81,16 +81,24 @@ export async function getNumTokensForChatMessages({
   return { numTokensTotal, numTokensPerMessage }
 }
 
+/**
+ * Converts a Zod schema to a JSON schema.
+ *
+ * @param options - object containing the `zodSchema` and `name`
+ * @returns JSON schema derived from the provided Zod schema
+ */
 export function zodToJsonSchema({
   zodSchema,
   name
 }: {
+  /** Zod schema */
   zodSchema: ZodTypeAny
+  /** Name of the schema */
   name: string
 }): any {
   const jsonSchema = zodToJsonSchemaImpl(zodSchema, {
     name,
-    $refStrategy: 'none'
+    $refStrategy: 'none' // ignores referencing
   })
 
   const parameters: any = jsonSchema.definitions?.[name]
@@ -103,6 +111,13 @@ export function zodToJsonSchema({
   return parameters
 }
 
+/**
+ * Generates a chat message function definition from a given task.
+ *
+ * @param task - task to generate a function definition from
+ * @returns function definition for a chat message
+ * @throws if the task name is invalid
+ */
 export function getChatMessageFunctionDefinitionFromTask(
   task: BaseTask<any, any>
 ): types.openai.ChatMessageFunction {
@@ -120,6 +135,12 @@ export function getChatMessageFunctionDefinitionFromTask(
   }
 }
 
+/**
+ * Generates an array of chat message function definitions from an array of tasks.
+ *
+ * @param tasks - array of tasks to generate function definitions from
+ * @returns array of function definitions for chat messages
+ */
 export function getChatMessageFunctionDefinitionsFromTasks(
   tasks: BaseTask<any, any>[]
 ): types.openai.ChatMessageFunction[] {

src/tokenizer.ts

@@ -48,6 +48,13 @@ export class TiktokenTokenizer implements Tokenizer {
 
 export const getTiktokenBPE = pMemoize(getTiktokenBPEImpl)
 
+/**
+ * Asynchronously retrieves the Byte Pair Encoding (BPE) for a specified Tiktoken encoding.
+ *
+ * @param encoding - Tiktoken encoding
+ * @param options - optional settings for the request
+ * @returns promise that resolves to the BPE for the specified encoding
+ */
 async function getTiktokenBPEImpl(
   encoding: TiktokenEncoding,
   {
@@ -64,6 +71,13 @@ async function getTiktokenBPEImpl(
   }).json<TiktokenBPE>()
 }
 
+/**
+ * Asynchronously creates and retrieves a tokenizer for a specified Tiktoken encoding.
+ *
+ * @param encoding - Tiktoken encoding
+ * @param options - optional settings for the request
+ * @returns promise resolving to a tokenizer for the specified encoding
+ */
 export async function getTokenizerForEncoding(
   encoding: TiktokenEncoding,
   options?: {
@@ -77,6 +91,13 @@ export async function getTokenizerForEncoding(
   return new TiktokenTokenizer(tiktoken)
 }
 
+/**
+ * Asynchronously creates and retrieves a tokenizer for a specified Tiktoken model.
+ *
+ * @param model - name of the Tiktoken model
+ * @param options - optional settings for the request
+ * @returns promise resolving to a tokenizer for the specified model
+ */
 export async function getTokenizerForModel(
   model: string,
   options?: {

src/url-utils.ts

@@ -7,6 +7,12 @@ const normalizedUrlCache = new QuickLRU<string, string | null>({
   maxSize: 4000
 })
 
+/**
+ * Checks if a URL is crawlable.
+ *
+ * @param url - URL string to check
+ * @returns whether the URL is crawlable
+ */
 export function isValidCrawlableUrl(url: string): boolean {
   try {
     if (!url || (isRelativeUrl(url) && !url.startsWith('//'))) {