☪

bin/scratch.ts

@@ -58,8 +58,9 @@ async function main() {
 
   const firecrawl = new FirecrawlClient()
   const res = await firecrawl.scrapeUrl({
-    // url: 'https://www.bbc.com/news/articles/cp4475gwny1o'
-    url: 'https://www.firecrawl.dev'
+    url: 'https://www.bbc.com/news/articles/cp4475gwny1o'
+    // url: 'https://www.theguardian.com/technology/article/2024/jun/04/openai-google-ai-risks-letter'
+    // url: 'https://www.firecrawl.dev'
   })
   console.log(JSON.stringify(res, null, 2))
 

src/services/firecrawl-client.ts

@@ -25,10 +25,33 @@ export namespace firecrawl {
    */
   export interface ScrapeResponse {
     success: boolean
-    data?: any
+    data?: Data
     error?: string
   }
 
+  export interface Data {
+    content?: string
+    markdown?: string
+    html?: string
+    metadata: Metadata
+  }
+
+  export interface Metadata {
+    title: string
+    description: string
+    keywords?: string
+    robots?: string
+    ogTitle?: string
+    ogDescription?: string
+    ogUrl?: string
+    ogImage?: string
+    ogLocaleAlternate?: any[]
+    ogSiteName?: string
+    sourceURL?: string
+    modifiedTime?: string
+    publishedTime?: string
+  }
+
   /**
    * Response interface for searching operations.
    */
@@ -132,9 +155,18 @@ export class FirecrawlClient extends AIFunctionsProvider {
       }
     }
 
-    return this.ky
-      .post('v0/scrapeUrl', { json })
+    const res = await this.ky
+      .post('v0/scrape', { json })
       .json<firecrawl.ScrapeResponse>()
+
+    if (!res.success || !res.data) return res
+
+    if (res.data.markdown) {
+      delete res.data.html
+      delete res.data.content
+    }
+
+    return res
   }
 
   async search(