From f36aa9b63a0fee0c54e7f835edf2a9e66ad9b629 Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Sun, 9 Jun 2024 02:18:00 -0500 Subject: [PATCH] feat: add social data client --- legacy/bin/scratch.ts | 17 +- legacy/readme.md | 2 +- legacy/src/services/index.ts | 1 + legacy/src/services/social-data-client.ts | 399 ++++++++++++++++++++++ 4 files changed, 412 insertions(+), 7 deletions(-) create mode 100644 legacy/src/services/social-data-client.ts diff --git a/legacy/bin/scratch.ts b/legacy/bin/scratch.ts index 2d503e86..11f7435b 100644 --- a/legacy/bin/scratch.ts +++ b/legacy/bin/scratch.ts @@ -18,7 +18,8 @@ import restoreCursor from 'restore-cursor' // } from '../src/services/twitter/index.js' // import { MidjourneyClient } from '../src/index.js' // import { BingClient } from '../src/index.js' -import { TavilyClient } from '../src/index.js' +// import { TavilyClient } from '../src/index.js' +import { SocialDataClient } from '../src/index.js' /** * Scratch pad for testing. @@ -112,11 +113,15 @@ async function main() { // }) // console.log(JSON.stringify(res, null, 2)) - const tavily = new TavilyClient() - const res = await tavily.search({ - query: 'when do experts predict that OpenAI will release GPT-5?', - include_answer: true - }) + // const tavily = new TavilyClient() + // const res = await tavily.search({ + // query: 'when do experts predict that OpenAI will release GPT-5?', + // include_answer: true + // }) + // console.log(JSON.stringify(res, null, 2)) + + const socialData = new SocialDataClient() + const res = await socialData.getUserByUsername('transitive_bs') console.log(JSON.stringify(res, null, 2)) } diff --git a/legacy/readme.md b/legacy/readme.md index 385343df..ce7023c7 100644 --- a/legacy/readme.md +++ b/legacy/readme.md @@ -150,6 +150,7 @@ Depending on the AI SDK and tool you want to use, you'll also need to install th | [SerpAPI](https://serpapi.com/search-api) | `SerpAPIClient` | Lightweight wrapper around SerpAPI for Google search. | | [Serper](https://serper.dev) | `SerperClient` | Lightweight wrapper around Serper for Google search. | | [Slack](https://api.slack.com/docs) | `SlackClient` | Send and receive Slack messages. | +| [SocialData](https://socialdata.tools) | `SocialDataClient` | Unofficial Twitter / X client (readonly) which is much cheaper than the official Twitter API. | | [Tavily](https://tavily.com) | `TavilyClient` | Web search API tailored for LLMs. | | [Twilio](https://www.twilio.com/docs/conversations/api) | `TwilioClient` | Twilio conversation API to send and receive SMS messages. | | [Twitter](https://developer.x.com/en/docs/twitter-api) | `TwitterClient` | Basic Twitter API methods for fetching users, tweets, and searching recent tweets. Includes support for plan-aware rate-limiting. Uses [Nango](https://www.nango.dev) for OAuth support. | @@ -198,7 +199,6 @@ See the [examples](./examples) directory for examples of how to use each of thes - [phantombuster](https://phantombuster.com) - [apify](https://apify.com/store) - perplexity - - [socialdata](https://socialdata.tools) - valtown - replicate - huggingface diff --git a/legacy/src/services/index.ts b/legacy/src/services/index.ts index ade60861..22c82b5e 100644 --- a/legacy/src/services/index.ts +++ b/legacy/src/services/index.ts @@ -16,6 +16,7 @@ export * from './searxng-client.js' export * from './serpapi-client.js' export * from './serper-client.js' export * from './slack-client.js' +export * from './social-data-client.js' export * from './tavily-client.js' export * from './twilio-client.js' export * from './weather-client.js' diff --git a/legacy/src/services/social-data-client.ts b/legacy/src/services/social-data-client.ts new file mode 100644 index 00000000..549b7e88 --- /dev/null +++ b/legacy/src/services/social-data-client.ts @@ -0,0 +1,399 @@ +import defaultKy, { type KyInstance } from 'ky' +import pThrottle from 'p-throttle' + +import { AIFunctionsProvider } from '../fns.js' +import { assert, getEnv, throttleKy } from '../utils.js' + +export namespace socialdata { + export const API_BASE_URL = 'https:///api.socialdata.tools' + + // Allow up to 120 requests per minute by default. + export const throttle = pThrottle({ + limit: 120, + interval: 60 * 1000 + }) + + export type GetTweetByIdOptions = { + id: string + } + + export type GetUsersByTweetByIdOptions = { + tweetId: string + cursor?: string + } + + export type SearchTweetOptions = { + query: string + cursor?: string + type?: 'Latest' | 'Top' + } + + export type SearchUsersOptions = { + query: string + } + + export type GetUserByIdOptions = { + userId: string + } + + export type GetUserByUsernameOptions = { + username: string + } + + export type GetUsersByIdOptions = { + userId: string + cursor?: string + } + + export type UserFollowingOptions = { + // The numeric ID of the desired follower. + sourceUserId: string + // The numeric ID of the desired user being followed. + targetUserId: string + // Maximum number of followers for target_user to look through. + maxCount?: number + } + + export type GetTweetsByUserIdOptions = { + userId: string + cursor?: string + replies?: boolean + } + + export type TweetResponse = Tweet | ErrorResponse + export type UserResponse = User | ErrorResponse + + export type UsersResponse = + | { + next_cursor: string + users: User[] + } + | ErrorResponse + + export type TweetsResponse = + | { + next_cursor: string + tweets: Tweet[] + } + | ErrorResponse + + export type UserFollowingResponse = UserFollowingStatus | ErrorResponse + + export interface ErrorResponse { + status: 'error' + message: string + } + + export interface Tweet { + tweet_created_at: string + id: number + id_str: string + text: any + full_text: string + source: string + truncated: boolean + in_reply_to_status_id: any + in_reply_to_status_id_str: any + in_reply_to_user_id: any + in_reply_to_user_id_str: any + in_reply_to_screen_name: any + user: User + quoted_status_id: any + quoted_status_id_str: any + is_quote_status: boolean + quoted_status: any + retweeted_status: any + quote_count: number + reply_count: number + retweet_count: number + favorite_count: number + lang: string + entities: Entities + views_count: number + bookmark_count: number + } + + export interface User { + id: number + id_str: string + name: string + screen_name: string + location: string + url: any + description: string + protected: boolean + verified: boolean + followers_count: number + friends_count: number + listed_count: number + favourites_count: number + statuses_count: number + created_at: string + profile_banner_url: string + profile_image_url_https: string + can_dm: boolean + } + + export interface Entities { + user_mentions?: any[] + urls?: any[] + hashtags?: any[] + symbols?: any[] + } + + export interface UserFollowingStatus { + status: string + source_user_id: string + target_user_id: string + is_following: boolean + followers_checked_count: number + } +} + +/** + * SocialData API is a scalable and reliable API that simplifies the process of + * fetching data from social media websites. At the moment, we only support X + * (formerly Twitter), but working on adding more integrations. + * + * With SocialData API, you can easily retrieve tweets, user profiles, user + * followers/following and other information without the need for proxies or + * parsing Twitter responses. This ensures a seamless and hassle-free + * integration with your application, saving you valuable time and effort. + * + * @see https://socialdata.tools + */ +export class SocialDataClient extends AIFunctionsProvider { + protected readonly ky: KyInstance + protected readonly apiKey: string + protected readonly apiBaseUrl: string + + constructor({ + apiKey = getEnv('SOCIAL_DATA_API_KEY'), + apiBaseUrl = socialdata.API_BASE_URL, + throttle = true, + ky = defaultKy + }: { + apiKey?: string + apiBaseUrl?: string + throttle?: boolean + ky?: KyInstance + } = {}) { + assert( + apiKey, + 'SocialDataClient missing required "apiKey" (defaults to "SOCIAL_DATA_API_KEY")' + ) + super() + + this.apiKey = apiKey + this.apiBaseUrl = apiBaseUrl + + const throttledKy = throttle ? throttleKy(ky, socialdata.throttle) : ky + + this.ky = throttledKy.extend({ + prefixUrl: this.apiBaseUrl, + headers: { + Authorization: `Bearer ${this.apiKey}` + } + }) + } + + /** + * Retrieve tweet details. + */ + async getTweetById(idOrOpts: string | socialdata.GetTweetByIdOptions) { + const options = typeof idOrOpts === 'string' ? { id: idOrOpts } : idOrOpts + + return this.ky + .get('twitter/statuses/show', { + searchParams: options + }) + .json() + } + + /** + * Retrieve all users who liked a tweet. + */ + async getUsersWhoLikedTweetById( + idOrOpts: string | socialdata.GetUsersByTweetByIdOptions + ) { + const { tweetId, ...params } = + typeof idOrOpts === 'string' ? { tweetId: idOrOpts } : idOrOpts + + return this.ky + .get(`twitter/tweets/${tweetId}/liking_users`, { + searchParams: params + }) + .json() + } + + /** + * Retrieve all users who retweeted a tweet. + */ + async getUsersWhoRetweetedTweetById( + idOrOpts: string | socialdata.GetUsersByTweetByIdOptions + ) { + const { tweetId, ...params } = + typeof idOrOpts === 'string' ? { tweetId: idOrOpts } : idOrOpts + + return this.ky + .get(`twitter/tweets/${tweetId}/retweeted_by`, { + searchParams: params + }) + .json() + } + + /** + * Returns array of tweets provided by Twitter search page. Typically Twitter + * returns ~20 results per page. You can request additional search results by + * sending another request to the same endpoint using cursor parameter. + * + * Search `type` defaults to `Top`. + */ + async searchTweets(queryOrOpts: string | socialdata.SearchTweetOptions) { + const options = + typeof queryOrOpts === 'string' ? { query: queryOrOpts } : queryOrOpts + + return this.ky + .get('twitter/search', { + searchParams: { + type: 'top', + ...options + } + }) + .json() + } + + /** + * Retrieve user profile details by user ID. + */ + async getUserById(idOrOpts: string | socialdata.GetUserByIdOptions) { + const { userId } = + typeof idOrOpts === 'string' ? { userId: idOrOpts } : idOrOpts + + return this.ky.get(`twitter/user/${userId}`).json() + } + + /** + * Retrieve user profile details by username. + */ + async getUserByUsername( + usernameOrOptions: string | socialdata.GetUserByUsernameOptions + ) { + const { username } = + typeof usernameOrOptions === 'string' + ? { username: usernameOrOptions } + : usernameOrOptions + + return this.ky + .get(`twitter/user/${username}`) + .json() + } + + /** + * Returns array of tweets from the user's tweets and replies timeline. + * Typically Twitter returns ~20 results per page. You can request additional + * search results by sending another request to the same endpoint using + * cursor parameter. + */ + async getTweetsByUserId( + idOrOpts: string | socialdata.GetTweetsByUserIdOptions + ) { + const { + userId, + replies = false, + ...params + } = typeof idOrOpts === 'string' ? { userId: idOrOpts } : idOrOpts + + return this.ky + .get( + `twitter/user/${userId}/${replies ? 'tweets-and-replies' : 'tweets'}`, + { + searchParams: params + } + ) + .json() + } + + /** + * Returns array of tweets from the user's likes timeline. Typically Twitter + * returns ~20 results per page. You can request additional search results + * by sending another request to the same endpoint using cursor parameter. + */ + async getTweetsLikedByUserId( + idOrOpts: string | socialdata.GetTweetsByUserIdOptions + ) { + const { userId, ...params } = + typeof idOrOpts === 'string' ? { userId: idOrOpts } : idOrOpts + + return this.ky + .get(`twitter/user/${userId}/likes`, { + searchParams: params + }) + .json() + } + + /** + * Retrieve user followers. + */ + async getFollowersForUserId( + idOrOpts: string | socialdata.GetUsersByIdOptions + ) { + const { userId: user_id, ...params } = + typeof idOrOpts === 'string' ? { userId: idOrOpts } : idOrOpts + + return this.ky + .get('twitter/followers/list', { + searchParams: { + user_id, + ...params + } + }) + .json() + } + + /** + * Retrieve user followers. + */ + async getFollowingForUserId( + idOrOpts: string | socialdata.GetUsersByIdOptions + ) { + const { userId: user_id, ...params } = + typeof idOrOpts === 'string' ? { userId: idOrOpts } : idOrOpts + + return this.ky + .get('twitter/friends/list', { + searchParams: { + user_id, + ...params + } + }) + .json() + } + + /** + * This endpoint provides a convenient way to check if a user is following + * another user. This will recursively retrieve all recent followers of + * target user (up to [max_count] total results) and check if the + * source_user_id is present among the retrieved followers. + */ + async isUserFollowingUser(opts: socialdata.UserFollowingOptions) { + const { sourceUserId, targetUserId, ...params } = opts + + return this.ky + .get(`twitter/user/${sourceUserId}/following/${targetUserId}`, { + searchParams: params + }) + .json() + } + + /** + * Returns a list of users with screenname or full name matching the search query. + */ + async searchUsersByUsername(queryOrOpts: socialdata.SearchUsersOptions) { + return this.ky + .get('twitter/search-users', { + searchParams: queryOrOpts + }) + .json() + } +}