modified fetchContentFromURL() to use node-fetch now that import is working

pull/73/head
Huda Joad 2023-11-24 23:38:02 +03:00
rodzic 2bad128834
commit 535ccfaefe
1 zmienionych plików z 11 dodań i 16 usunięć

Wyświetl plik

@ -2,23 +2,18 @@ const https = require('https'); // Import for webscraping (fetchContentFromURL(u
import { OpenAIApi, Configuration } from 'openai';
import { fetch } from 'node-fetch';
// Placeholder function to fetch content from URL using a web scraping service
// Function to fetch content from URL using a web scraping service
async function fetchContentFromURL(url) {
return new Promise((resolve, reject) => {
https.get(url, (response) => {
let data = '';
// A chunk of data has been received.
response.on('data', (chunk) => {
data += chunk;
});
// The whole response has been received.
response.on('end', () => {
resolve(data);
});
}).on("error", (error) => {
reject(`Error fetching URL: ${error.message}`);
});
});
try {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
return await response.text();
} catch (error) {
console.error(`Could not fetch content from URL: ${error}`);
throw error;
}
}
function simplifyContent(content) {