kopia lustrzana https://github.com/learn-awesome/learndb
modified fetchContentFromURL() to use node-fetch now that import is working
rodzic
2bad128834
commit
535ccfaefe
|
@ -2,23 +2,18 @@ const https = require('https'); // Import for webscraping (fetchContentFromURL(u
|
|||
import { OpenAIApi, Configuration } from 'openai';
|
||||
import { fetch } from 'node-fetch';
|
||||
|
||||
// Placeholder function to fetch content from URL using a web scraping service
|
||||
// Function to fetch content from URL using a web scraping service
|
||||
async function fetchContentFromURL(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
https.get(url, (response) => {
|
||||
let data = '';
|
||||
// A chunk of data has been received.
|
||||
response.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
// The whole response has been received.
|
||||
response.on('end', () => {
|
||||
resolve(data);
|
||||
});
|
||||
}).on("error", (error) => {
|
||||
reject(`Error fetching URL: ${error.message}`);
|
||||
});
|
||||
});
|
||||
try {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
return await response.text();
|
||||
} catch (error) {
|
||||
console.error(`Could not fetch content from URL: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function simplifyContent(content) {
|
||||
|
|
Ładowanie…
Reference in New Issue