🎌

package.json

@@ -66,9 +66,14 @@
     "@nangohq/node": "^0.39.32",
     "dedent": "^1.5.3",
     "delay": "^6.0.0",
+    "hash-object": "^5.0.1",
+    "is-relative-url": "^4.0.0",
     "jsonrepair": "^3.6.1",
     "ky": "^1.2.4",
+    "normalize-url": "^8.0.1",
+    "p-map": "^7.0.2",
     "p-throttle": "^6.1.0",
+    "quick-lru": "^7.0.0",
     "twitter-api-sdk": "^1.2.1",
     "type-fest": "^4.18.3",
     "zod": "^3.23.3",
@@ -105,8 +110,8 @@
     "@dexaai/dexter": "^2.0.3",
     "@genkit-ai/ai": "^0.5.2",
     "@langchain/core": "^0.2.5",
-    "expr-eval": "^2.0.2",
+    "ai": "^3.1.22",
-    "ai": "^3.1.22"
+    "expr-eval": "^2.0.2"
   },
   "peerDependenciesMeta": {
     "@dexaai/dexter": {

pnpm-lock.yaml

@@ -20,15 +20,30 @@ importers:
       delay:
         specifier: ^6.0.0
         version: 6.0.0
+      hash-object:
+        specifier: ^5.0.1
+        version: 5.0.1
+      is-relative-url:
+        specifier: ^4.0.0
+        version: 4.0.0
       jsonrepair:
         specifier: ^3.6.1
         version: 3.8.0
       ky:
         specifier: ^1.2.4
         version: 1.3.0
+      normalize-url:
+        specifier: ^8.0.1
+        version: 8.0.1
+      p-map:
+        specifier: ^7.0.2
+        version: 7.0.2
       p-throttle:
         specifier: ^6.1.0
         version: 6.1.0
+      quick-lru:
+        specifier: ^7.0.0
+        version: 7.0.0
       twitter-api-sdk:
         specifier: ^1.2.1
         version: 1.2.1
@@ -2320,6 +2335,10 @@ packages:
     resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==}
     engines: {node: '>= 0.10'}
 
+  is-absolute-url@4.0.1:
+    resolution: {integrity: sha512-/51/TKE88Lmm7Gc4/8btclNXWS+g50wXhYJq8HWIBAGUBnoAdRu1aXeh364t/O7wXDAcTJDP8PNuNKWUDWie+A==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+
   is-any-array@2.0.1:
     resolution: {integrity: sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ==}
 
@@ -2492,6 +2511,10 @@ packages:
     resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==}
     engines: {node: '>= 0.4'}
 
+  is-relative-url@4.0.0:
+    resolution: {integrity: sha512-PkzoL1qKAYXNFct5IKdKRH/iBQou/oCC85QhXj6WKtUQBliZ4Yfd3Zk27RHu9KQG8r6zgvAA2AQKC9p+rqTszg==}
+    engines: {node: '>=14.16'}
+
   is-scoped@3.0.0:
     resolution: {integrity: sha512-ezxLUq30kiTvP0w/5n9tj4qTOKlrA07Oty1hwTQ+lcqw11x6uc8sp7VRb2OVGRzKfCHZ2A22T5Zsau/Q2Akb0g==}
     engines: {node: '>=12'}
@@ -3529,6 +3552,10 @@ packages:
     resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==}
     engines: {node: '>=10'}
 
+  quick-lru@7.0.0:
+    resolution: {integrity: sha512-MX8gB7cVYTrYcFfAnfLlhRd0+Toyl8yX8uBx1MrX7K0jegiz9TumwOK27ldXrgDlHRdVi+MqU9Ssw6dr4BNreg==}
+    engines: {node: '>=18'}
+
   ramda@0.29.1:
     resolution: {integrity: sha512-OfxIeWzd4xdUNxlWhgFazxsA/nl3mS4/jGZI5n00uWOoSSFRhC1b6gl6xvmzUamgmqELraWp0J/qqVlXYPDPyA==}
 
@@ -7005,6 +7032,8 @@ snapshots:
 
   ipaddr.js@1.9.1: {}
 
+  is-absolute-url@4.0.1: {}
+
   is-any-array@2.0.1: {}
 
   is-array-buffer@3.0.4:
@@ -7140,6 +7169,10 @@ snapshots:
       call-bind: 1.0.7
       has-tostringtag: 1.0.2
 
+  is-relative-url@4.0.0:
+    dependencies:
+      is-absolute-url: 4.0.1
+
   is-scoped@3.0.0:
     dependencies:
       scoped-regex: 3.0.0
@@ -8088,6 +8121,8 @@ snapshots:
 
   quick-lru@5.1.1: {}
 
+  quick-lru@7.0.0: {}
+
   ramda@0.29.1: {}
 
   range-parser@1.2.1: {}

readme.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <a href="https://trywalter.ai"><img alt="Agentic" src="/media/agentic-header.jpg" width="616"></a>
+  <a href="https://trywalter.ai"><img alt="Agentic" src="/media/agentic-header.jpg" width="308"></a>
 </p>
 
 <p align="center">
@@ -154,7 +154,6 @@ The SDK-specific imports are all isolated to keep the main `@agentic/stdlib` as
   - instructor-js
   - TODO
 - services
-  - calculator
   - e2b
   - search-and-scrape
   - replicate
@@ -166,6 +165,8 @@ The SDK-specific imports are all isolated to keep the main `@agentic/stdlib` as
     - provide a converter for langchain `DynamicStructuredTool`
   - pull from other libs
   - pull from [nango](https://docs.nango.dev/integrations/overview)
+- tools
+  - calculator
 - tools / chains / flows / runnables
   - market maps
 - https://github.com/causaly/zod-validation-error

src/url-utils.ts

@@ -0,0 +1,99 @@
+import isRelativeUrlImpl from 'is-relative-url'
+import normalizeUrlImpl, { type Options } from 'normalize-url'
+import QuickLRU from 'quick-lru'
+
+import { hashObject } from './utils.js'
+
+const protocolAllowList = new Set(['https:', 'http:'])
+const normalizedUrlCache = new QuickLRU<string, string | undefined>({
+  maxSize: 4000
+})
+
+/**
+ * Checks if a URL is crawlable.
+ *
+ * @param url - URL string to check
+ * @returns whether the URL is crawlable
+ */
+export function isValidCrawlableUrl(url: string): boolean {
+  try {
+    if (!url || isRelativeUrl(url)) {
+      return false
+    }
+
+    const parsedUrl = new URL(url)
+    if (!protocolAllowList.has(parsedUrl.protocol)) {
+      return false
+    }
+
+    const normalizedUrl = normalizeUrl(url)
+    if (!normalizedUrl) {
+      return false
+    }
+
+    return true
+  } catch {
+    return false
+  }
+}
+
+export function isRelativeUrl(url: string): boolean {
+  if (!url || typeof url !== 'string') return false
+
+  return isRelativeUrlImpl(url) && !url.startsWith('//')
+}
+
+/**
+ * Normalizes a URL string.
+ *
+ * @param url - URL string to normalize
+ * @param options - options for normalization.
+ * @returns normalized URL string or null if an invalid URL was passed
+ */
+export function normalizeUrl(
+  url: string,
+  options?: Options
+): string | undefined {
+  let normalizedUrl: string | undefined
+  let cacheKey: string | undefined
+
+  try {
+    if (!url || isRelativeUrl(url)) {
+      return
+    }
+
+    const opts = {
+      stripWWW: false,
+      defaultProtocol: 'https',
+      normalizeProtocol: true,
+      forceHttps: false,
+      stripHash: false,
+      stripTextFragment: true,
+      removeQueryParameters: [/^utm_\w+/i, 'ref', 'ref_src'],
+      removeTrailingSlash: true,
+      removeSingleSlash: true,
+      removeExplicitPort: true,
+      sortQueryParameters: true,
+      ...options
+    } as Required<Options>
+
+    const optionsHash = hashObject(opts)
+    cacheKey = `${url}-${optionsHash}`
+    normalizedUrl = normalizedUrlCache.get(cacheKey)
+
+    if (normalizedUrl !== undefined) {
+      return normalizedUrl
+    }
+
+    normalizedUrl = normalizeUrlImpl(url, opts)
+  } catch {
+    // ignore invalid urls
+    normalizedUrl = undefined
+  }
+
+  if (cacheKey) {
+    normalizedUrlCache.set(cacheKey, normalizedUrl!)
+  }
+
+  return normalizedUrl
+}

src/utils.ts

@@ -1,5 +1,6 @@
 import type { Jsonifiable } from 'type-fest'
 import dedent from 'dedent'
+import hashObjectImpl from 'hash-object'
 
 import type * as types from './types.js'
 
@@ -140,3 +141,7 @@ const dedenter = dedent.withOptions({ escapeSpecialCharacters: true })
 export function cleanStringForModel(text: string): string {
   return dedenter(text).trim()
 }
+
+export function hashObject(object: Record<string, any>): string {
+  return hashObjectImpl(object, { algorithm: 'sha256' })
+}