From 9d67337b03300ef319953a184bedd59d6f1613aa Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Tue, 4 Jun 2024 22:10:19 -0500 Subject: [PATCH] feat: add twitter client --- legacy/bin/scratch.ts | 23 +- legacy/package.json | 25 +- legacy/readme.md | 24 +- legacy/src/index.ts | 2 +- legacy/src/nango.ts | 47 ++ legacy/src/services/twitter-client.ts | 111 ----- legacy/src/services/twitter/client.ts | 78 ++++ legacy/src/services/twitter/error.ts | 32 ++ legacy/src/services/twitter/index.ts | 5 + legacy/src/services/twitter/twitter-client.ts | 405 ++++++++++++++++++ legacy/src/services/twitter/types.ts | 71 +++ legacy/src/services/twitter/utils.ts | 140 ++++++ legacy/tsup.config.ts | 3 + 13 files changed, 835 insertions(+), 131 deletions(-) create mode 100644 legacy/src/nango.ts delete mode 100644 legacy/src/services/twitter-client.ts create mode 100644 legacy/src/services/twitter/client.ts create mode 100644 legacy/src/services/twitter/error.ts create mode 100644 legacy/src/services/twitter/index.ts create mode 100644 legacy/src/services/twitter/twitter-client.ts create mode 100644 legacy/src/services/twitter/types.ts create mode 100644 legacy/src/services/twitter/utils.ts diff --git a/legacy/bin/scratch.ts b/legacy/bin/scratch.ts index aeb79262..bdc394d8 100644 --- a/legacy/bin/scratch.ts +++ b/legacy/bin/scratch.ts @@ -11,7 +11,11 @@ import restoreCursor from 'restore-cursor' // import { FirecrawlClient } from '../src/index.js' // import { ExaClient } from '../src/index.js' // import { DiffbotClient } from '../src/index.js' -import { WolframClient } from '../src/index.js' +// import { WolframClient } from '../src/index.js' +import { + createTwitterV2Client, + TwitterClient +} from '../src/services/twitter/index.js' /** * Scratch pad for testing. @@ -76,14 +80,19 @@ async function main() { // }) // console.log(JSON.stringify(res, null, 2)) - const wolfram = new WolframClient() - // const res = await diffbot.analyzeUrl({ - // url: 'https://www.bbc.com/news/articles/cp4475gwny1o' + // const wolfram = new WolframClient() + // const res = await wolfram.ask({ + // input: 'population of new york city' // }) - const res = await wolfram.ask({ - input: 'population of new york city' + // console.log(res) + + const client = await createTwitterV2Client({ + // scopes: ['tweet.read', 'users.read', 'offline.access'] }) - console.log(res) + const twitter = new TwitterClient({ client }) + + const user = await twitter.findUserByUsername({ username: 'transitive_bs' }) + console.log(user) } try { diff --git a/legacy/package.json b/legacy/package.json index 7bffa173..fd470dc4 100644 --- a/legacy/package.json +++ b/legacy/package.json @@ -38,10 +38,25 @@ "import": "./dist/sdks/genkit.js", "default": "./dist/sdks/genkit.js" }, + "./langchain": { + "types": "./dist/sdks/langchain.d.ts", + "import": "./dist/sdks/langchain.js", + "default": "./dist/sdks/langchain.js" + }, + "./llamaindex": { + "types": "./dist/sdks/llamaindex.d.ts", + "import": "./dist/sdks/llamaindex.js", + "default": "./dist/sdks/llamaindex.js" + }, "./calculator": { "types": "./dist/tools/calculator.d.ts", "import": "./dist/tools/calculator.js", "default": "./dist/tools/calculator.js" + }, + "./twitter": { + "types": "./dist/services/twitter/index.d.ts", + "import": "./dist/services/twitter/index.js", + "default": "./dist/services/twitter/index.js" } }, "files": [ @@ -74,7 +89,6 @@ "p-map": "^7.0.2", "p-throttle": "^6.1.0", "quick-lru": "^7.0.0", - "twitter-api-sdk": "^1.2.1", "type-fest": "^4.18.3", "zod": "^3.23.3", "zod-to-json-schema": "^3.23.0" @@ -104,6 +118,7 @@ "ts-node": "^10.9.2", "tsup": "^8.0.2", "tsx": "^4.11.0", + "twitter-api-sdk": "^1.2.1", "typescript": "^5.4.5", "vitest": "2.0.0-beta.3" }, @@ -113,7 +128,8 @@ "@langchain/core": "^0.2.5", "ai": "^3.1.22", "expr-eval": "^2.0.2", - "llamaindex": "^0.3.15" + "llamaindex": "^0.3.15", + "twitter-api-sdk": "^1.2.1" }, "peerDependenciesMeta": { "@dexaai/dexter": { @@ -125,13 +141,16 @@ "@langchain/core": { "optional": true }, + "ai": { + "optional": true + }, "expr-eval": { "optional": true }, "llamaindex": { "optional": true }, - "ai": { + "twitter-api-sdk": { "optional": true } }, diff --git a/legacy/readme.md b/legacy/readme.md index 1382e28d..b42e42bd 100644 --- a/legacy/readme.md +++ b/legacy/readme.md @@ -116,9 +116,10 @@ The SDK-specific imports are all isolated to keep the main `@agentic/stdlib` as - SDK adaptors should be as lightweight as possible and be optional peer dependencies of `@agentic/stdlib` - SDK adatptor entrypoints should all be isolated to their own top-level imports - `@agentic/stdlib/ai-sdk` + - `@agentic/stdlib/langchain` + - `@agentic/stdlib/llamaindex` - `@agentic/stdlib/dexter` - `@agentic/stdlib/genkit` - - `@agentic/stdlib/langchain` ## Services @@ -136,23 +137,28 @@ The SDK-specific imports are all isolated to keep the main `@agentic/stdlib` as - serpapi - serper - twitter (WIP) +- wolfram alpha - weatherapi - wikipedia ## AI SDKs -- openai sdk -- vercel ai sdk -- dexa dexter -- firebase genkit -- langchain -- llamaindex +- OpenAI SDK + - no need for an adaptor; use `AIFunctionSet.specs` or `AIFunctionSet.toolSpecs` +- Vercel AI SDK + - `import { createAISDKTools } from '@agentic/stdlib/ai-sdk'` +- LangChain + - `import { createLangChainTools } from '@agentic/stdlib/langchain'` +- LlamaIndex + - `import { createLlamaIndexTools } from '@agentic/stdlib/llamaindex'` +- Firebase Genkit + - `import { createGenkitTools } from '@agentic/stdlib/genkit'` +- Dexa Dexter + - `import { createDexterFunctions } from '@agentic/stdlib/dexter'` ## TODO - rename this repo to agentic -- sdks - - TODO - services - e2b - search-and-scrape diff --git a/legacy/src/index.ts b/legacy/src/index.ts index 50edc27c..d3e155fc 100644 --- a/legacy/src/index.ts +++ b/legacy/src/index.ts @@ -1,9 +1,9 @@ export * from './ai-function-set.js' export * from './create-ai-function.js' -export * from './create-ai-function.js' export * from './errors.js' export * from './fns.js' export * from './message.js' +export * from './nango.js' export * from './parse-structured-output.js' export * from './services/index.js' export type * from './types.js' diff --git a/legacy/src/nango.ts b/legacy/src/nango.ts new file mode 100644 index 00000000..3dbb0e6a --- /dev/null +++ b/legacy/src/nango.ts @@ -0,0 +1,47 @@ +import { type Connection, Nango } from '@nangohq/node' + +import { getEnv } from './utils.js' + +// This is intentionally left as a global singleton to avoid re-creating the +// Nango connection instance on successive calls in serverless environments. +let _nango: Nango | null = null + +export function getNango(): Nango { + if (!_nango) { + const secretKey = getEnv('NANGO_SECRET_KEY')?.trim() + if (!secretKey) { + throw new Error(`Missing required "NANGO_SECRET_KEY"`) + } + + _nango = new Nango({ secretKey }) + } + + return _nango +} + +export function validateNangoConnectionOAuthScopes({ + connection, + scopes +}: { + connection: Connection + scopes: string[] +}) { + const connectionScopes = new Set( + connection.credentials.raw.scope.split(' ') + ) + const missingScopes = new Set() + + for (const scope of scopes) { + if (!connectionScopes.has(scope)) { + missingScopes.add(scope) + } + } + + if (missingScopes.size > 0) { + throw new Error( + `Nango connection ${connection.id} is missing required OAuth scopes: ${[ + ...missingScopes.values() + ].join(', ')}` + ) + } +} diff --git a/legacy/src/services/twitter-client.ts b/legacy/src/services/twitter-client.ts deleted file mode 100644 index 668d9661..00000000 --- a/legacy/src/services/twitter-client.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { Nango } from '@nangohq/node' -import { auth, Client as TwitterClient } from 'twitter-api-sdk' - -import { assert, getEnv } from '../utils.js' - -// The Twitter+Nango client auth connection key -const nangoTwitterProviderConfigKey = 'twitter-v2' - -// The Twitter OAuth2User class requires a client id, which we don't have -// since we're using Nango for auth, so instead we just pass a dummy value -// and allow Nango to handle all auth/refresh/access token management. -const twitterClientId = 'xbot' - -const defaultRequiredTwitterOAuthScopes = new Set([ - 'tweet.read', - 'users.read', - 'offline.access', - 'tweet.write' -]) - -let _nango: Nango | null = null - -function getNango(): Nango { - if (!_nango) { - const secretKey = getEnv('NANGO_SECRET_KEY')?.trim() - if (!secretKey) { - throw new Error(`Missing required "NANGO_SECRET_KEY"`) - } - - _nango = new Nango({ secretKey }) - } - - return _nango -} - -async function getTwitterAuth({ - scopes, - nangoConnectionId, - nangoCallbackUrl -}: { - scopes: Set - nangoConnectionId: string - nangoCallbackUrl: string -}): Promise { - const nango = getNango() - const connection = await nango.getConnection( - nangoTwitterProviderConfigKey, - nangoConnectionId - ) - - // console.debug('nango twitter connection', connection) - // connection.credentials.raw - // { - // token_type: 'bearer', - // expires_in: number, - // access_token: string - // scope: string - // expires_at: string - // } - const connectionScopes = new Set( - connection.credentials.raw.scope.split(' ') - ) - const missingScopes = new Set() - - for (const scope of scopes) { - if (!connectionScopes.has(scope)) { - missingScopes.add(scope) - } - } - - if (missingScopes.size > 0) { - throw new Error( - `Nango connection ${nangoConnectionId} is missing required OAuth scopes: ${[ - ...missingScopes.values() - ].join(', ')}` - ) - } - - const token = connection.credentials.raw - assert(token) - - return new auth.OAuth2User({ - client_id: twitterClientId, - callback: nangoCallbackUrl, - scopes: [...scopes.values()] as any, - token - }) -} - -export async function getTwitterClient({ - scopes = defaultRequiredTwitterOAuthScopes, - nangoConnectionId = getEnv('NANGO_CONNECTION_ID'), - nangoCallbackUrl = getEnv('NANGO_CALLBACK_URL') -}: { - scopes?: Set - nangoConnectionId?: string - nangoCallbackUrl?: string -} = {}): Promise { - assert(nangoConnectionId, 'twitter client missing nangoConnectionId') - assert(nangoCallbackUrl, 'twitter client missing nangoCallbackUrl') - - // NOTE: Nango handles refreshing the oauth access token for us - const twitterAuth = await getTwitterAuth({ - scopes, - nangoConnectionId, - nangoCallbackUrl - }) - - // Twitter API v2 using OAuth 2.0 - return new TwitterClient(twitterAuth) -} diff --git a/legacy/src/services/twitter/client.ts b/legacy/src/services/twitter/client.ts new file mode 100644 index 00000000..1ba99601 --- /dev/null +++ b/legacy/src/services/twitter/client.ts @@ -0,0 +1,78 @@ +import { auth, Client as TwitterV2Client } from 'twitter-api-sdk' + +import { getNango, validateNangoConnectionOAuthScopes } from '../../nango.js' +import { assert, getEnv } from '../../utils.js' + +// Auth new Nango accounts here: https://app.nango.dev/connections + +// The Twitter OAuth2User class requires a client id, which we don't have +// since we're using Nango for auth, so instead we just pass a dummy value +// and allow Nango to handle all auth/refresh/access token management. +const dummyTwitterClientId = 'agentic' + +export const defaultTwitterOAuthScopes = [ + 'tweet.read', + 'users.read', + 'offline.access', + 'tweet.write' +] + +async function createTwitterAuth({ + scopes, + nangoConnectionId, + nangoCallbackUrl, + nangoProviderConfigKey +}: { + scopes: string[] + nangoConnectionId: string + nangoCallbackUrl: string + nangoProviderConfigKey: string +}): Promise { + const nango = getNango() + const connection = await nango.getConnection( + nangoProviderConfigKey, + nangoConnectionId + ) + + validateNangoConnectionOAuthScopes({ + connection, + scopes + }) + + const token = connection.credentials.raw + assert(token) + + return new auth.OAuth2User({ + client_id: dummyTwitterClientId, + callback: nangoCallbackUrl, + scopes: scopes as any[], + token + }) +} + +export async function createTwitterV2Client({ + scopes = defaultTwitterOAuthScopes, + nangoConnectionId = getEnv('NANGO_CONNECTION_ID'), + nangoCallbackUrl = getEnv('NANGO_CALLBACK_URL') ?? + 'https://api.nango.dev/oauth/callback', + nangoProviderConfigKey = 'twitter-v2' +}: { + scopes?: string[] + nangoConnectionId?: string + nangoCallbackUrl?: string + nangoProviderConfigKey?: string +} = {}): Promise { + assert(nangoConnectionId, 'twitter client missing nangoConnectionId') + assert(nangoCallbackUrl, 'twitter client missing nangoCallbackUrl') + + // NOTE: Nango handles refreshing the oauth access token for us + const twitterAuth = await createTwitterAuth({ + scopes, + nangoConnectionId, + nangoCallbackUrl, + nangoProviderConfigKey + }) + + // Twitter API v2 using OAuth 2.0 + return new TwitterV2Client(twitterAuth) +} diff --git a/legacy/src/services/twitter/error.ts b/legacy/src/services/twitter/error.ts new file mode 100644 index 00000000..4fc8de56 --- /dev/null +++ b/legacy/src/services/twitter/error.ts @@ -0,0 +1,32 @@ +export type TwitterErrorType = + | 'twitter:forbidden' + | 'twitter:auth' + | 'twitter:rate-limit' + | 'twitter:unknown' + | 'network' + +export class TwitterError extends Error { + type: TwitterErrorType + isFinal: boolean + status?: number + + constructor( + message: string, + { + type, + isFinal = false, + status, + ...opts + }: ErrorOptions & { + type: TwitterErrorType + isFinal?: boolean + status?: number + } + ) { + super(message, opts) + + this.type = type + this.isFinal = isFinal + this.status = status ?? (opts.cause as any)?.status + } +} diff --git a/legacy/src/services/twitter/index.ts b/legacy/src/services/twitter/index.ts new file mode 100644 index 00000000..a002d7f4 --- /dev/null +++ b/legacy/src/services/twitter/index.ts @@ -0,0 +1,5 @@ +export * from './client.js' +export * from './error.js' +export * from './twitter-client.js' +export type * from './types.js' +export * from './utils.js' diff --git a/legacy/src/services/twitter/twitter-client.ts b/legacy/src/services/twitter/twitter-client.ts new file mode 100644 index 00000000..7d72c2eb --- /dev/null +++ b/legacy/src/services/twitter/twitter-client.ts @@ -0,0 +1,405 @@ +import pThrottle from 'p-throttle' +import { z } from 'zod' + +import type * as types from './types.js' +import { aiFunction, AIFunctionsProvider } from '../../fns.js' +import { assert, getEnv } from '../../utils.js' +import { handleKnownTwitterErrors } from './utils.js' + +/** + * This file contains rate-limited wrappers around all of the core Twitter API + * methods that this project uses. + * + * NOTE: Twitter has different API rate limits and quotas per plan, so in order + * to rate-limit effectively, our throttles need to either use the lowest common + * denominator OR vary based on the twitter developer plan you're using. We + * chose to go with the latter. + * + * @see https://developer.twitter.com/en/docs/twitter-api/rate-limits + */ + +type TwitterApiMethod = + | 'createTweet' + | 'usersIdMentions' + | 'findTweetById' + | 'findTweetsById' + | 'findUserById' + | 'findUserByUsername' + +const TWENTY_FOUR_HOURS_MS = 24 * 60 * 60 * 1000 +const FIFTEEN_MINUTES_MS = 15 * 60 * 1000 + +const twitterApiRateLimitsByPlan: Record< + types.TwitterApiPlan, + Record< + TwitterApiMethod, + { + readonly limit: number + readonly interval: number + } + > +> = { + free: { + // 50 per 24h per user + // 50 per 24h per app + createTweet: { limit: 50, interval: TWENTY_FOUR_HOURS_MS }, + + // TODO: according to the twitter docs, this shouldn't be allowed on the + // free plan, but it seems to work... + usersIdMentions: { limit: 1, interval: FIFTEEN_MINUTES_MS }, + + // TODO: according to the twitter docs, this shouldn't be allowed on the + // free plan, but it seems to work... + findTweetById: { limit: 1, interval: FIFTEEN_MINUTES_MS }, + + // TODO: according to the twitter docs, this shouldn't be allowed on the + // free plan, but it seems to work... + findTweetsById: { limit: 1, interval: FIFTEEN_MINUTES_MS }, + + findUserById: { limit: 1, interval: FIFTEEN_MINUTES_MS }, + findUserByUsername: { limit: 1, interval: FIFTEEN_MINUTES_MS } + }, + + basic: { + // 100 per 24h per user + // 1667 per 24h per app + createTweet: { limit: 100, interval: TWENTY_FOUR_HOURS_MS }, + + // https://developer.twitter.com/en/docs/twitter-api/tweets/timelines/api-reference/get-users-id-mentions + // TODO: undocumented + // 180 per 15m per user + // 450 per 15m per app + usersIdMentions: { limit: 180, interval: FIFTEEN_MINUTES_MS }, + + // 15 per 15m per user + // 15 per 15m per app + findTweetById: { limit: 15, interval: FIFTEEN_MINUTES_MS }, + findTweetsById: { limit: 15, interval: FIFTEEN_MINUTES_MS }, + + findUserById: { limit: 100, interval: TWENTY_FOUR_HOURS_MS }, + findUserByUsername: { limit: 100, interval: TWENTY_FOUR_HOURS_MS } + }, + + pro: { + // 100 per 15m per user + // 10k per 24h per app + createTweet: { limit: 100, interval: FIFTEEN_MINUTES_MS }, + + // 180 per 15m per user + // 450 per 15m per app + usersIdMentions: { limit: 180, interval: FIFTEEN_MINUTES_MS }, + + // TODO: why would the per-user rate-limit be less than the per-app one?! + // 900 per 15m per user + // 450 per 15m per app + findTweetById: { limit: 450, interval: FIFTEEN_MINUTES_MS }, + findTweetsById: { limit: 450, interval: FIFTEEN_MINUTES_MS }, + + findUserById: { limit: 300, interval: FIFTEEN_MINUTES_MS }, + findUserByUsername: { limit: 300, interval: FIFTEEN_MINUTES_MS } + }, + + enterprise: { + // NOTE: these are just placeholders; the enterprise plan seems to be + // completely customizable, but it's still useful to define rate limits + // for robustness. These values just 10x those of the pro plan. + createTweet: { limit: 1000, interval: FIFTEEN_MINUTES_MS }, + usersIdMentions: { limit: 1800, interval: FIFTEEN_MINUTES_MS }, + findTweetById: { limit: 4500, interval: FIFTEEN_MINUTES_MS }, + findTweetsById: { limit: 4500, interval: FIFTEEN_MINUTES_MS }, + findUserById: { limit: 3000, interval: FIFTEEN_MINUTES_MS }, + findUserByUsername: { limit: 3000, interval: FIFTEEN_MINUTES_MS } + } +} + +export class TwitterClient extends AIFunctionsProvider { + readonly client: types.TwitterV2Client + readonly twitterApiPlan: types.TwitterApiPlan + + constructor({ + client, + twitterApiPlan = (getEnv('TWITTER_API_PLAN') as types.TwitterApiPlan) ?? + 'free' + }: { + client: types.TwitterV2Client + twitterApiPlan?: types.TwitterApiPlan + }) { + assert( + client, + 'TwitterClient missing required "client" which should be an instance of "twitter-api-sdk" (use `getTwitterV2Client` to initialize the underlying V2 Twitter SDK using Nango OAuth)' + ) + assert(twitterApiPlan, 'TwitterClient missing required "twitterApiPlan"') + + super() + + this.client = client + this.twitterApiPlan = twitterApiPlan + + const twitterApiRateLimits = twitterApiRateLimitsByPlan[twitterApiPlan]! + assert(twitterApiRateLimits, `Invalid twitter api plan: ${twitterApiPlan}`) + + const createTweetThrottle = pThrottle(twitterApiRateLimits.createTweet) + const findTweetByIdThrottle = pThrottle(twitterApiRateLimits.findTweetById) + const findTweetsByIdThrottle = pThrottle( + twitterApiRateLimits.findTweetsById + ) + const findUserByIdThrottle = pThrottle(twitterApiRateLimits.findUserById) + const findUserByUsernameThrottle = pThrottle( + twitterApiRateLimits.findUserByUsername + ) + + this._createTweet = createTweetThrottle(createTweetImpl(this.client)) + this._findTweetById = findTweetByIdThrottle(findTweetByIdImpl(this.client)) + this._findTweetsById = findTweetsByIdThrottle( + findTweetsByIdImpl(this.client) + ) + this._findUserById = findUserByIdThrottle(findUserByIdImpl(this.client)) + this._findUserByUsername = findUserByUsernameThrottle( + findUserByUsernameImpl(this.client) + ) + } + + protected _createTweet: ReturnType + protected _findTweetById: ReturnType + protected _findTweetsById: ReturnType + protected _findUserById: ReturnType + protected _findUserByUsername: ReturnType + + @aiFunction({ + name: 'create_tweet', + description: 'Creates a new tweet', + inputSchema: z.object({ + text: z.string().min(1) + }) + }) + async createTweet( + params: types.CreateTweetParams + ): Promise { + return this._createTweet(params) + } + + @aiFunction({ + name: 'get_tweet_by_id', + description: 'Fetch a tweet by its ID', + inputSchema: z.object({ + id: z.string().min(1) + }) + }) + async findTweetById({ + id, + ...params + }: { id: string } & types.FindTweetByIdParams) { + assert( + this.twitterApiPlan !== 'free', + 'TwitterClient.findTweetById not supported on free plan' + ) + + return this._findTweetById(id, params) + } + + @aiFunction({ + name: 'get_tweets_by_id', + description: 'Fetch an array of tweets by their IDs', + inputSchema: z.object({ + ids: z.array(z.string().min(1)) + }) + }) + async findTweetsById({ ids, ...params }: types.FindTweetsByIdParams) { + assert( + this.twitterApiPlan !== 'free', + 'TwitterClient.findTweetsById not supported on free plan' + ) + + return this._findTweetsById(ids, params) + } + + @aiFunction({ + name: 'get_twitter_user_by_id', + description: 'Fetch a twitter user by ID', + inputSchema: z.object({ + id: z.string().min(1) + }) + }) + async findUserById({ + id, + ...params + }: { id: string } & types.FindUserByIdParams) { + assert( + this.twitterApiPlan !== 'free', + 'TwitterClient.findUserById not supported on free plan' + ) + + return this._findUserById(id, params) + } + + @aiFunction({ + name: 'get_twitter_user_by_username', + description: 'Fetch a twitter user by username', + inputSchema: z.object({ + username: z.string().min(1) + }) + }) + async findUserByUsername({ + username, + ...params + }: { username: string } & types.FindUserByUsernameParams) { + assert( + this.twitterApiPlan !== 'free', + 'TwitterClient.findUserByUsername not supported on free plan' + ) + + return this._findUserByUsername(username, params) + } +} + +const defaultTwitterQueryTweetFields: types.TwitterQueryTweetFields = [ + 'attachments', + 'author_id', + 'conversation_id', + 'created_at', + 'entities', + 'geo', + 'id', + 'in_reply_to_user_id', + 'lang', + 'public_metrics', + 'possibly_sensitive', + 'referenced_tweets', + 'text' + // 'context_annotations', // not needed (way too verbose and noisy) + // 'edit_controls', / not needed + // 'non_public_metrics', // don't have access to + // 'organic_metrics', // don't have access to + // 'promoted_metrics, // don't have access to + // 'reply_settings', / not needed + // 'source', // not needed + // 'withheld' // not needed +] + +const defaultTwitterQueryUserFields: types.TwitterQueryUserFields = [ + 'created_at', + 'description', + 'entities', + 'id', + 'location', + 'name', + 'pinned_tweet_id', + 'profile_image_url', + 'protected', + 'public_metrics', + 'url', + 'username', + 'verified' + // 'most_recent_tweet_id', + // 'verified_type', + // 'withheld' +] + +const defaultTweetQueryParams: types.TweetsQueryOptions = { + // https://developer.twitter.com/en/docs/twitter-api/expansions + expansions: [ + 'author_id', + 'in_reply_to_user_id', + 'referenced_tweets.id', + 'referenced_tweets.id.author_id', + 'entities.mentions.username', + // TODO + 'attachments.media_keys', + 'geo.place_id', + 'attachments.poll_ids' + ], + 'tweet.fields': defaultTwitterQueryTweetFields, + 'user.fields': defaultTwitterQueryUserFields +} + +const defaultUserQueryParams: types.TwitterUserQueryOptions = { + // https://developer.twitter.com/en/docs/twitter-api/expansions + expansions: ['pinned_tweet_id'], + 'tweet.fields': defaultTwitterQueryTweetFields, + 'user.fields': defaultTwitterQueryUserFields +} + +function createTweetImpl(client: types.TwitterV2Client) { + return async ( + params: types.CreateTweetParams + ): Promise => { + try { + const { data: tweet } = await client.tweets.createTweet(params) + + if (!tweet?.id) { + throw new Error('invalid createTweet response') + } + + return tweet + } catch (err: any) { + console.error('error creating tweet', JSON.stringify(err, null, 2)) + + handleKnownTwitterErrors(err, { label: 'creating tweet' }) + throw err + } + } +} + +function findTweetByIdImpl(client: types.TwitterV2Client) { + return async (tweetId: string, params?: types.FindTweetByIdParams) => { + try { + return await client.tweets.findTweetById(tweetId, { + ...defaultTweetQueryParams, + ...params + }) + } catch (err: any) { + handleKnownTwitterErrors(err, { label: `fetching tweet ${tweetId}` }) + throw err + } + } +} + +function findTweetsByIdImpl(client: types.TwitterV2Client) { + return async ( + ids: string[], + params?: Omit + ) => { + try { + return await client.tweets.findTweetsById({ + ...defaultTweetQueryParams, + ...params, + ids + }) + } catch (err: any) { + handleKnownTwitterErrors(err, { label: `fetching ${ids.length} tweets` }) + throw err + } + } +} + +function findUserByIdImpl(client: types.TwitterV2Client) { + return async (userId: string, params?: types.FindUserByIdParams) => { + try { + return await client.users.findUserById(userId, { + ...defaultUserQueryParams, + ...params + }) + } catch (err: any) { + handleKnownTwitterErrors(err, { + label: `fetching user with id ${userId}` + }) + throw err + } + } +} + +function findUserByUsernameImpl(client: types.TwitterV2Client) { + return async (username: string, params?: types.FindUserByUsernameParams) => { + try { + return await client.users.findUserByUsername(username, { + ...defaultUserQueryParams, + ...params + }) + } catch (err: any) { + handleKnownTwitterErrors(err, { + label: `fetching user with username ${username}` + }) + throw err + } + } +} diff --git a/legacy/src/services/twitter/types.ts b/legacy/src/services/twitter/types.ts new file mode 100644 index 00000000..4ee2893d --- /dev/null +++ b/legacy/src/services/twitter/types.ts @@ -0,0 +1,71 @@ +import type { AsyncReturnType, Simplify } from 'type-fest' +import { type Client as TwitterV2Client } from 'twitter-api-sdk' + +export { type Client as TwitterV2Client } from 'twitter-api-sdk' + +export type TwitterApiPlan = 'free' | 'basic' | 'pro' | 'enterprise' + +export type TweetsQueryOptions = Simplify< + Pick< + Parameters[0], + 'expansions' | 'tweet.fields' | 'user.fields' + > +> + +export type TwitterUserQueryOptions = Simplify< + Pick< + NonNullable[1]>, + 'expansions' | 'tweet.fields' | 'user.fields' + > +> + +export type TwitterQueryTweetFields = TweetsQueryOptions['tweet.fields'] +export type TwitterQueryUserFields = TweetsQueryOptions['user.fields'] + +export type TwitterUserIdMentionsQueryOptions = Simplify< + NonNullable[1]> +> + +export type CreateTweetParams = Simplify< + Parameters[0] +> + +export type UsersIdMentionsParams = Simplify< + Parameters[1] +> + +export type FindTweetByIdParams = Simplify< + Parameters[1] +> + +export type FindTweetsByIdParams = Simplify< + Parameters[0] +> + +export type FindUserByIdParams = Simplify< + Parameters[1] +> + +export type FindUserByUsernameParams = Simplify< + Parameters[1] +> + +type Unpacked = T extends (infer U)[] ? U : T + +export type Tweet = Simplify< + NonNullable< + Unpacked< + AsyncReturnType['data'] + > + > +> +export type TwitterUser = Simplify< + NonNullable['data']> +> +export type CreatedTweet = Simplify< + NonNullable['data']> +> + +export type TwitterUrl = Simplify< + Unpacked['urls']>> +> diff --git a/legacy/src/services/twitter/utils.ts b/legacy/src/services/twitter/utils.ts new file mode 100644 index 00000000..a4a97dd0 --- /dev/null +++ b/legacy/src/services/twitter/utils.ts @@ -0,0 +1,140 @@ +import type * as types from './types.js' +import { omit } from '../../utils.js' +import { TwitterError } from './error.js' + +/** + * Error handler which takes in an unknown Error object and converts it to a + * structured TwitterError object for a set of common Twitter API errors. + * + * Re-throws the error and will never return. + */ +export function handleKnownTwitterErrors( + err: any, + { label = '' }: { label?: string } = {} +) { + if (err.status === 403) { + // user may have deleted the tweet we're trying to respond to + throw new TwitterError( + err.error?.detail || `error ${label}: 403 forbidden`, + { + type: 'twitter:forbidden', + isFinal: true, + cause: err + } + ) + } else if (err.status === 401) { + throw new TwitterError(`error ${label}: unauthorized`, { + type: 'twitter:auth', + cause: err + }) + } else if (err.status === 400) { + if ( + /value passed for the token was invalid/i.test( + err.error?.error_description + ) + ) { + throw new TwitterError(`error ${label}: invalid auth token`, { + type: 'twitter:auth', + cause: err + }) + } + } else if (err.status === 429) { + throw new TwitterError(`error ${label}: too many requests`, { + type: 'twitter:rate-limit', + cause: err + }) + } else if (err.status === 404) { + throw new TwitterError(err.toString(), { + type: 'twitter:forbidden', + isFinal: true, + cause: err + }) + } + + if (err.status >= 400 && err.status < 500) { + throw new TwitterError( + `error ${label}: ${err.status} ${ + err.error?.description || err.toString() + }`, + { + type: 'twitter:unknown', + isFinal: true, + cause: err + } + ) + } else if (err.status >= 500) { + throw new TwitterError( + `error ${label}: ${err.status} ${ + err.error?.description || err.toString() + }`, + { + type: 'twitter:unknown', + isFinal: false, + cause: err + } + ) + } + + const reason = err.toString().toLowerCase() + + if (reason.includes('fetcherror') || reason.includes('enotfound')) { + throw new TwitterError(err.toString(), { + type: 'network', + cause: err + }) + } + + // Otherwise, propagate the original error + throw err +} + +export function getPrunedTweet( + tweet: Partial +): Partial { + const urls = tweet.entities?.urls + let text = tweet.text + if (text && urls) { + for (const url of urls) { + if (!url.expanded_url || !url.url) continue + text = text!.replaceAll(url.url, url.expanded_url!) + } + } + + return { + ...omit( + tweet, + 'conversation_id', + 'public_metrics', + 'created_at', + 'entities', + 'possibly_sensitive' + ), + text + } +} + +export function getPrunedTwitterUser( + twitterUser: Partial +): Partial { + const urls = twitterUser.entities?.description?.urls + let description = twitterUser.description + if (description && urls) { + for (const url of urls) { + if (!url.expanded_url || !url.url) continue + description = description!.replaceAll(url.url, url.expanded_url!) + } + } + + return { + ...omit( + twitterUser, + 'public_metrics', + 'created_at', + 'verified', + 'protected', + 'url', + 'entities' + ), + description + } +} diff --git a/legacy/tsup.config.ts b/legacy/tsup.config.ts index 30d2242d..5fad85e4 100644 --- a/legacy/tsup.config.ts +++ b/legacy/tsup.config.ts @@ -7,6 +7,9 @@ export default defineConfig([ 'src/sdks/ai-sdk.ts', 'src/sdks/dexter.ts', 'src/sdks/genkit.ts', + 'src/sdks/langchain.ts', + 'src/sdks/llamaindex.ts', + 'src/services/twitter/index.ts', 'src/tools/calculator.ts' ], outDir: 'dist',