From 8fb9ddf27117d05045e02ea1d6602fecdadcd4dc Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Sun, 2 Jun 2024 19:35:48 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=A4=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/services/searxng-client.ts | 456 +++++++++++++++++---------------- src/services/serpapi-client.ts | 12 +- src/services/serper-client.ts | 4 +- 3 files changed, 253 insertions(+), 219 deletions(-) diff --git a/src/services/searxng-client.ts b/src/services/searxng-client.ts index 97adc00..1103af2 100644 --- a/src/services/searxng-client.ts +++ b/src/services/searxng-client.ts @@ -1,222 +1,229 @@ import defaultKy, { type KyInstance } from 'ky' +import { z } from 'zod' +import { aiFunction, AIFunctionsProvider } from '../fns.js' import { assert, getEnv, omit, pick, pruneUndefined } from '../utils.js' export namespace searxng { - export type SearchCategory = - | 'general' - | 'images' - | 'videos' - | 'news' - | 'map' - | 'music' - | 'it' - | 'science' - | 'files' - | 'social media' + export const SearchCategorySchema = z.enum([ + 'general', + 'images', + 'videos', + 'news', + 'map', + 'music', + 'it', + 'science', + 'files', + 'social media' + ]) + export type SearchCategory = z.infer - export type SearchEngine = - | '9gag' - | 'annas archive' - | 'apk mirror' - | 'apple app store' - | 'ahmia' - | 'anaconda' - | 'arch linux wiki' - | 'artic' - | 'arxiv' - | 'ask' - | 'bandcamp' - | 'wikipedia' - | 'bilibili' - | 'bing' - | 'bing images' - | 'bing news' - | 'bing videos' - | 'bitbucket' - | 'bpb' - | 'btdigg' - | 'ccc-tv' - | 'openverse' - | 'chefkoch' - | 'crossref' - | 'crowdview' - | 'yep' - | 'yep images' - | 'yep news' - | 'curlie' - | 'currency' - | 'bahnhof' - | 'deezer' - | 'destatis' - | 'deviantart' - | 'ddg definitions' - | 'docker hub' - | 'erowid' - | 'wikidata' - | 'duckduckgo' - | 'duckduckgo images' - | 'duckduckgo videos' - | 'duckduckgo news' - | 'duckduckgo weather' - | 'apple maps' - | 'emojipedia' - | 'tineye' - | 'etymonline' - | '1x' - | 'fdroid' - | 'flickr' - | 'free software directory' - | 'frinkiac' - | 'fyyd' - | 'genius' - | 'gentoo' - | 'gitlab' - | 'github' - | 'codeberg' - | 'goodreads' - | 'google' - | 'google images' - | 'google news' - | 'google videos' - | 'google scholar' - | 'google play apps' - | 'google play movies' - | 'material icons' - | 'gpodder' - | 'habrahabr' - | 'hackernews' - | 'hoogle' - | 'imdb' - | 'imgur' - | 'ina' - | 'invidious' - | 'jisho' - | 'kickass' - | 'lemmy communities' - | 'lemmy users' - | 'lemmy posts' - | 'lemmy comments' - | 'library genesis' - | 'z-library' - | 'library of congress' - | 'lingva' - | 'lobste.rs' - | 'mastodon users' - | 'mastodon hashtags' - | 'mdn' - | 'metacpan' - | 'mixcloud' - | 'mozhi' - | 'mwmbl' - | 'npm' - | 'nyaa' - | 'mankier' - | 'odysee' - | 'openairedatasets' - | 'openairepublications' - | 'openstreetmap' - | 'openrepos' - | 'packagist' - | 'pdbe' - | 'photon' - | 'pinterest' - | 'piped' - | 'piped.music' - | 'piratebay' - | 'podcastindex' - | 'presearch' - | 'presearch images' - | 'presearch videos' - | 'presearch news' - | 'pub.dev' - | 'pubmed' - | 'pypi' - | 'qwant' - | 'qwant news' - | 'qwant images' - | 'qwant videos' - | 'radio browser' - | 'reddit' - | 'rottentomatoes' - | 'sepiasearch' - | 'soundcloud' - | 'stackoverflow' - | 'askubuntu' - | 'internetarchivescholar' - | 'superuser' - | 'searchcode code' - | 'semantic scholar' - | 'startpage' - | 'tokyotoshokan' - | 'solidtorrents' - | 'tagesschau' - | 'tmdb' - | 'torch' - | 'unsplash' - | 'yandex music' - | 'yahoo' - | 'yahoo news' - | 'youtube' - | 'dailymotion' - | 'vimeo' - | 'wiby' - | 'alexandria' - | 'wikibooks' - | 'wikinews' - | 'wikiquote' - | 'wikisource' - | 'wikispecies' - | 'wiktionary' - | 'wikiversity' - | 'wikivoyage' - | 'wikicommons.images' - | 'wolframalpha' - | 'dictzone' - | 'mymemory translated' - | '1337x' - | 'duden' - | 'seznam' - | 'mojeek' - | 'moviepilot' - | 'naver' - | 'rubygems' - | 'peertube' - | 'mediathekviewweb' - | 'yacy' - | 'yacy images' - | 'rumble' - | 'livespace' - | 'wordnik' - | 'woxikon.de synonyme' - | 'seekr news' - | 'seekr images' - | 'seekr videos' - | 'sjp.pwn' - | 'stract' - | 'svgrepo' - | 'tootfinder' - | 'wallhaven' - | 'wikimini' - | 'wttr.in' - | 'yummly' - | 'brave' - | 'brave.images' - | 'brave.videos' - | 'brave.news' - | 'lib.rs' - | 'sourcehut' - | 'goo' - | 'bt4g' - | 'pkg.go.dev' + export const SearchEngineSchema = z.enum([ + '9gag', + 'annas archive', + 'apk mirror', + 'apple app store', + 'ahmia', + 'anaconda', + 'arch linux wiki', + 'artic', + 'arxiv', + 'ask', + 'bandcamp', + 'wikipedia', + 'bilibili', + 'bing', + 'bing images', + 'bing news', + 'bing videos', + 'bitbucket', + 'bpb', + 'btdigg', + 'ccc-tv', + 'openverse', + 'chefkoch', + 'crossref', + 'crowdview', + 'yep', + 'yep images', + 'yep news', + 'curlie', + 'currency', + 'bahnhof', + 'deezer', + 'destatis', + 'deviantart', + 'ddg definitions', + 'docker hub', + 'erowid', + 'wikidata', + 'duckduckgo', + 'duckduckgo images', + 'duckduckgo videos', + 'duckduckgo news', + 'duckduckgo weather', + 'apple maps', + 'emojipedia', + 'tineye', + 'etymonline', + '1x', + 'fdroid', + 'flickr', + 'free software directory', + 'frinkiac', + 'fyyd', + 'genius', + 'gentoo', + 'gitlab', + 'github', + 'codeberg', + 'goodreads', + 'google', + 'google images', + 'google news', + 'google videos', + 'google scholar', + 'google play apps', + 'google play movies', + 'material icons', + 'gpodder', + 'habrahabr', + 'hackernews', + 'hoogle', + 'imdb', + 'imgur', + 'ina', + 'invidious', + 'jisho', + 'kickass', + 'lemmy communities', + 'lemmy users', + 'lemmy posts', + 'lemmy comments', + 'library genesis', + 'z-library', + 'library of congress', + 'lingva', + 'lobste.rs', + 'mastodon users', + 'mastodon hashtags', + 'mdn', + 'metacpan', + 'mixcloud', + 'mozhi', + 'mwmbl', + 'npm', + 'nyaa', + 'mankier', + 'odysee', + 'openairedatasets', + 'openairepublications', + 'openstreetmap', + 'openrepos', + 'packagist', + 'pdbe', + 'photon', + 'pinterest', + 'piped', + 'piped.music', + 'piratebay', + 'podcastindex', + 'presearch', + 'presearch images', + 'presearch videos', + 'presearch news', + 'pub.dev', + 'pubmed', + 'pypi', + 'qwant', + 'qwant news', + 'qwant images', + 'qwant videos', + 'radio browser', + 'reddit', + 'rottentomatoes', + 'sepiasearch', + 'soundcloud', + 'stackoverflow', + 'askubuntu', + 'internetarchivescholar', + 'superuser', + 'searchcode code', + 'semantic scholar', + 'startpage', + 'tokyotoshokan', + 'solidtorrents', + 'tagesschau', + 'tmdb', + 'torch', + 'unsplash', + 'yandex music', + 'yahoo', + 'yahoo news', + 'youtube', + 'dailymotion', + 'vimeo', + 'wiby', + 'alexandria', + 'wikibooks', + 'wikinews', + 'wikiquote', + 'wikisource', + 'wikispecies', + 'wiktionary', + 'wikiversity', + 'wikivoyage', + 'wikicommons.images', + 'wolframalpha', + 'dictzone', + 'mymemory translated', + '1337x', + 'duden', + 'seznam', + 'mojeek', + 'moviepilot', + 'naver', + 'rubygems', + 'peertube', + 'mediathekviewweb', + 'yacy', + 'yacy images', + 'rumble', + 'livespace', + 'wordnik', + 'woxikon.de synonyme', + 'seekr news', + 'seekr images', + 'seekr videos', + 'sjp.pwn', + 'stract', + 'svgrepo', + 'tootfinder', + 'wallhaven', + 'wikimini', + 'wttr.in', + 'yummly', + 'brave', + 'brave.images', + 'brave.videos', + 'brave.news', + 'lib.rs', + 'sourcehut', + 'goo', + 'bt4g', + 'pkg.go.dev' + ]) + export type SearchEngine = z.infer - export interface SearchOptions { - query: string - categories?: SearchCategory[] - engines?: SearchEngine[] - language?: string - pageno?: number - } + export const SearchOptionsSchema = z.object({ + query: z.string().describe('search query'), + categories: z.array(SearchCategorySchema).optional(), + engines: z.array(SearchEngineSchema).optional(), + language: z.string().optional(), + pageno: z.number().int().optional() + }) + export type SearchOptions = z.infer export interface SearchResult { title: string @@ -246,7 +253,7 @@ export namespace searxng { * * See [perplexica](https://github.com/ItzCrazyKns/Perplexica/blob/master/docker-compose.yaml) for an example. */ -export class SearxngClient { +export class SearxngClient extends AIFunctionsProvider { readonly ky: KyInstance readonly apiBaseUrl: string @@ -254,7 +261,6 @@ export class SearxngClient { apiBaseUrl = getEnv('SEARXNG_API_BASE_URL'), ky = defaultKy }: { - apiKey?: string apiBaseUrl?: string ky?: KyInstance } = {}) { @@ -262,12 +268,34 @@ export class SearxngClient { apiBaseUrl, 'SearxngClient missing required "apiBaseUrl" (defaults to "SEARXNG_API_BASE_URL")' ) + super() this.apiBaseUrl = apiBaseUrl this.ky = ky.extend({ prefixUrl: apiBaseUrl }) } + @aiFunction({ + name: 'searxng', + description: `Searches across multiple search engines using a local instance of Searxng. To search only specific engines, use the \`engines\` parameter. + +The most important search engines are: + +- "reddit" (Reddit posts) +- "google" (Google web search) +- "google news" (Google News search) +- "brave" (Brave web search) +- "arxiv" (academic papers) +- "genius" (Genius.com for song lyrics) +- "imdb" (movies and TV shows) +- "hackernews" (Hacker News) +- "wikidata" (Wikidata) +- "wolframalpha" (Wolfram Alpha) +- "youtube" (YouTube videos) +- "github" (GitHub code and repositories) +`, + inputSchema: searxng.SearchOptionsSchema + }) async search({ query, ...opts diff --git a/src/services/serpapi-client.ts b/src/services/serpapi-client.ts index f351f07..b84eab9 100644 --- a/src/services/serpapi-client.ts +++ b/src/services/serpapi-client.ts @@ -649,7 +649,7 @@ export class SerpAPIClient extends AIFunctionsProvider { apiBaseUrl?: string ky?: KyInstance } & serpapi.ClientParams = {}) { - assert(apiKey, `Error SerpAPIClient missing required "apiKey"`) + assert(apiKey, 'Error SerpAPIClient missing required "apiKey"') super() this.apiKey = apiKey @@ -667,7 +667,13 @@ export class SerpAPIClient extends AIFunctionsProvider { 'Uses Google Search to return the most relevant web pages for a given query. Can also be used to find up-to-date news and information about many topics.', inputSchema: z.object({ q: z.string().describe('search query'), - num: z.number().int().positive().default(5).optional() + num: z + .number() + .int() + .positive() + .default(5) + .optional() + .describe('number of results to return') }) }) async search(queryOrOpts: string | serpapi.GoogleParameters) { @@ -685,7 +691,7 @@ export class SerpAPIClient extends AIFunctionsProvider { ...rest, engine: 'google', api_key: this.apiKey, - ...(options as any) // TODO + ...(options as any) }, timeout }) diff --git a/src/services/serper-client.ts b/src/services/serper-client.ts index 9fb227b..b4df6df 100644 --- a/src/services/serper-client.ts +++ b/src/services/serper-client.ts @@ -227,7 +227,7 @@ export class SerperClient extends AIFunctionsProvider { } & serper.ClientParams = {}) { assert( apiKey, - `SerperClient missing required "apiKey" (defaults to "SERPER_API_KEY" env var)` + 'SerperClient missing required "apiKey" (defaults to "SERPER_API_KEY" env var)' ) super() @@ -239,7 +239,7 @@ export class SerperClient extends AIFunctionsProvider { this.ky = ky.extend({ prefixUrl: this.apiBaseUrl, headers: { - 'X-API-KEY': this.apiKey + 'x-api-key': this.apiKey } }) }