kopia lustrzana https://github.com/Stopka/fedicrawl
Not existing feeds are deleted from databse
rodzic
cefbd76066
commit
3ed37c04ce
|
@ -3469,9 +3469,9 @@
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
"node_modules/follow-redirects": {
|
"node_modules/follow-redirects": {
|
||||||
"version": "1.14.5",
|
"version": "1.14.9",
|
||||||
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz",
|
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
|
||||||
"integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==",
|
"integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==",
|
||||||
"funding": [
|
"funding": [
|
||||||
{
|
{
|
||||||
"type": "individual",
|
"type": "individual",
|
||||||
|
@ -6344,9 +6344,10 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/minimist": {
|
"node_modules/minimist": {
|
||||||
"version": "1.2.5",
|
"version": "1.2.6",
|
||||||
"dev": true,
|
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
|
||||||
"license": "MIT"
|
"integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
|
||||||
|
"dev": true
|
||||||
},
|
},
|
||||||
"node_modules/ms": {
|
"node_modules/ms": {
|
||||||
"version": "2.1.2",
|
"version": "2.1.2",
|
||||||
|
@ -11163,9 +11164,9 @@
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
"follow-redirects": {
|
"follow-redirects": {
|
||||||
"version": "1.14.5",
|
"version": "1.14.9",
|
||||||
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz",
|
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
|
||||||
"integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA=="
|
"integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w=="
|
||||||
},
|
},
|
||||||
"form-data": {
|
"form-data": {
|
||||||
"version": "3.0.1",
|
"version": "3.0.1",
|
||||||
|
@ -13249,7 +13250,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"minimist": {
|
"minimist": {
|
||||||
"version": "1.2.5",
|
"version": "1.2.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
|
||||||
|
"integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
"ms": {
|
"ms": {
|
||||||
|
|
|
@ -26,16 +26,16 @@
|
||||||
"migrate:resolve": "npx prisma migrate resolve",
|
"migrate:resolve": "npx prisma migrate resolve",
|
||||||
"prisma:generate": "npx prisma generate",
|
"prisma:generate": "npx prisma generate",
|
||||||
"prisma:studio": "npx prisma studio",
|
"prisma:studio": "npx prisma studio",
|
||||||
"start:deploy": "npm run migrate:deploy && npm run start"
|
"start:deploy": "npm run migrate:deploy && npm run start"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@prisma/client": "^3.6.0",
|
"@prisma/client": "^3.6.0",
|
||||||
"axios": "^0.21.1",
|
"axios": "^0.21.1",
|
||||||
"npmlog": "^6.0.0",
|
"npmlog": "^6.0.0",
|
||||||
"typescript-collections": "^1.3.3",
|
|
||||||
"zod": "^3.11.6",
|
|
||||||
"rimraf": "^3.0.2",
|
"rimraf": "^3.0.2",
|
||||||
"striptags": "^3.2.0"
|
"striptags": "^3.2.0",
|
||||||
|
"typescript-collections": "^1.3.3",
|
||||||
|
"zod": "^3.11.6"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/jest": "^27.0.2",
|
"@types/jest": "^27.0.2",
|
||||||
|
@ -49,11 +49,11 @@
|
||||||
"eslint-plugin-node": "^11.1.0",
|
"eslint-plugin-node": "^11.1.0",
|
||||||
"eslint-plugin-promise": "^5.1.1",
|
"eslint-plugin-promise": "^5.1.1",
|
||||||
"eslint-plugin-react": "^7.27.1",
|
"eslint-plugin-react": "^7.27.1",
|
||||||
|
"jest": "^27.3.0",
|
||||||
"prisma": "^3.6.0",
|
"prisma": "^3.6.0",
|
||||||
"standard": "*",
|
"standard": "*",
|
||||||
"typescript": "^4.3.5",
|
|
||||||
"ts-jest": "^27.0.7",
|
"ts-jest": "^27.0.7",
|
||||||
"jest": "^27.3.0"
|
"typescript": "^4.3.5"
|
||||||
},
|
},
|
||||||
"jest": {
|
"jest": {
|
||||||
"moduleFileExtensions": [
|
"moduleFileExtensions": [
|
||||||
|
|
|
@ -0,0 +1,193 @@
|
||||||
|
/*
|
||||||
|
Warnings:
|
||||||
|
|
||||||
|
- A unique constraint covering the columns `[name,nodeId]` on the table `Feed` will be added. If there are existing duplicate values, this will fail.
|
||||||
|
- A unique constraint covering the columns `[domain]` on the table `Node` will be added. If there are existing duplicate values, this will fail.
|
||||||
|
- A unique constraint covering the columns `[name]` on the table `Tag` will be added. If there are existing duplicate values, this will fail.
|
||||||
|
|
||||||
|
*/
|
||||||
|
-- DropForeignKey
|
||||||
|
ALTER TABLE "Email" DROP CONSTRAINT "Email_feedId_fkey";
|
||||||
|
|
||||||
|
-- DropForeignKey
|
||||||
|
ALTER TABLE "Feed" DROP CONSTRAINT "Feed_nodeId_fkey";
|
||||||
|
|
||||||
|
-- DropForeignKey
|
||||||
|
ALTER TABLE "FeedToTag" DROP CONSTRAINT "FeedToTag_feedId_fkey";
|
||||||
|
|
||||||
|
-- DropForeignKey
|
||||||
|
ALTER TABLE "FeedToTag" DROP CONSTRAINT "FeedToTag_tagId_fkey";
|
||||||
|
|
||||||
|
-- DropForeignKey
|
||||||
|
ALTER TABLE "Field" DROP CONSTRAINT "Field_feedId_fkey";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Email_address_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Feed_bot_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Feed_createdAt_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Feed_description_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Feed_displayName_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Feed_fulltext_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Feed_lastStatusAt_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Feed_locked_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Feed_name_nodeId_key";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Feed_parentFeedName_parentFeedDomain_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Feed_refreshedAt_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Feed_type_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Field_name_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Field_value_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Node_domain_key";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Node_foundAt_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Node_halfYearActiveUserCount_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Node_monthActiveUserCount_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Node_openRegistrations_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Node_refreshAttemptedAt_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Node_refreshedAt_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Node_softwareName_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Node_softwareVersion_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Node_statusesCount_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Node_totalUserCount_idx";
|
||||||
|
|
||||||
|
-- DropIndex
|
||||||
|
DROP INDEX "Tag_name_key";
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Email_address_idx" ON "Email"("address");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Feed_displayName_idx" ON "Feed"("displayName");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Feed_description_idx" ON "Feed"("description");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Feed_bot_idx" ON "Feed"("bot");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Feed_locked_idx" ON "Feed"("locked");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Feed_lastStatusAt_idx" ON "Feed"("lastStatusAt");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Feed_createdAt_idx" ON "Feed"("createdAt");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Feed_refreshedAt_idx" ON "Feed"("refreshedAt");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Feed_parentFeedName_parentFeedDomain_idx" ON "Feed"("parentFeedName", "parentFeedDomain");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Feed_type_idx" ON "Feed"("type");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Feed_fulltext_idx" ON "Feed"("fulltext");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "Feed_name_nodeId_key" ON "Feed"("name", "nodeId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Field_name_idx" ON "Field"("name");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Field_value_idx" ON "Field"("value");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "Node_domain_key" ON "Node"("domain");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Node_softwareName_idx" ON "Node"("softwareName");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Node_softwareVersion_idx" ON "Node"("softwareVersion");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Node_totalUserCount_idx" ON "Node"("totalUserCount");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Node_monthActiveUserCount_idx" ON "Node"("monthActiveUserCount");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Node_halfYearActiveUserCount_idx" ON "Node"("halfYearActiveUserCount");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Node_statusesCount_idx" ON "Node"("statusesCount");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Node_openRegistrations_idx" ON "Node"("openRegistrations");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Node_refreshedAt_idx" ON "Node"("refreshedAt");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Node_refreshAttemptedAt_idx" ON "Node"("refreshAttemptedAt");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "Node_foundAt_idx" ON "Node"("foundAt");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "Tag_name_key" ON "Tag"("name");
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "Email" ADD CONSTRAINT "Email_feedId_fkey" FOREIGN KEY ("feedId") REFERENCES "Feed"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "FeedToTag" ADD CONSTRAINT "FeedToTag_feedId_fkey" FOREIGN KEY ("feedId") REFERENCES "Feed"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "FeedToTag" ADD CONSTRAINT "FeedToTag_tagId_fkey" FOREIGN KEY ("tagId") REFERENCES "Tag"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "Field" ADD CONSTRAINT "Field_feedId_fkey" FOREIGN KEY ("feedId") REFERENCES "Feed"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "Feed" ADD CONSTRAINT "Feed_nodeId_fkey" FOREIGN KEY ("nodeId") REFERENCES "Node"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
|
@ -5,7 +5,7 @@ datasource db {
|
||||||
|
|
||||||
generator client {
|
generator client {
|
||||||
provider = "prisma-client-js"
|
provider = "prisma-client-js"
|
||||||
previewFeatures = ["extendedIndexes","fullTextSearch"]
|
previewFeatures = ["extendedIndexes","fullTextSearch","referentialActions"]
|
||||||
}
|
}
|
||||||
|
|
||||||
model Tag {
|
model Tag {
|
||||||
|
@ -17,16 +17,16 @@ model Tag {
|
||||||
model Email {
|
model Email {
|
||||||
id String @id @default(uuid()) @db.Uuid
|
id String @id @default(uuid()) @db.Uuid
|
||||||
address String
|
address String
|
||||||
feed Feed @relation(fields: [feedId], references: [id])
|
feed Feed @relation(fields: [feedId], references: [id], onDelete: Cascade)
|
||||||
feedId String @db.Uuid
|
feedId String @db.Uuid
|
||||||
|
|
||||||
@@index([address])
|
@@index([address])
|
||||||
}
|
}
|
||||||
|
|
||||||
model FeedToTag {
|
model FeedToTag {
|
||||||
feed Feed @relation(fields: [feedId], references: [id])
|
feed Feed @relation(fields: [feedId], references: [id], onDelete: Cascade)
|
||||||
feedId String @db.Uuid
|
feedId String @db.Uuid
|
||||||
tag Tag @relation(fields: [tagId], references: [id])
|
tag Tag @relation(fields: [tagId], references: [id], onDelete: Cascade)
|
||||||
tagId String @db.Uuid
|
tagId String @db.Uuid
|
||||||
|
|
||||||
@@id([feedId, tagId])
|
@@id([feedId, tagId])
|
||||||
|
@ -36,7 +36,7 @@ model Field {
|
||||||
id String @id @default(uuid()) @db.Uuid
|
id String @id @default(uuid()) @db.Uuid
|
||||||
name String
|
name String
|
||||||
value String
|
value String
|
||||||
feed Feed @relation(fields: [feedId], references: [id])
|
feed Feed @relation(fields: [feedId], references: [id], onDelete: Cascade)
|
||||||
feedId String @db.Uuid
|
feedId String @db.Uuid
|
||||||
|
|
||||||
@@index([name])
|
@@index([name])
|
||||||
|
@ -50,7 +50,7 @@ enum FeedType{
|
||||||
|
|
||||||
model Feed {
|
model Feed {
|
||||||
id String @id @default(uuid()) @db.Uuid
|
id String @id @default(uuid()) @db.Uuid
|
||||||
node Node @relation(fields: [nodeId], references: [id])
|
node Node @relation(fields: [nodeId], references: [id], onDelete: Cascade)
|
||||||
nodeId String @db.Uuid
|
nodeId String @db.Uuid
|
||||||
foundAt DateTime @default(now())
|
foundAt DateTime @default(now())
|
||||||
refreshedAt DateTime @updatedAt
|
refreshedAt DateTime @updatedAt
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
export class NoSupportedLinkError extends Error {
|
||||||
|
public constructor (domain:string) {
|
||||||
|
super(`No supported link node info link for ${domain}`)
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,11 +1,12 @@
|
||||||
import { retrieveWellKnown } from './retrieveWellKnown'
|
import { retrieveWellKnown } from './retrieveWellKnown'
|
||||||
import { retrieveNodeInfo, NodeInfo } from './retrieveNodeInfo'
|
import { retrieveNodeInfo, NodeInfo } from './retrieveNodeInfo'
|
||||||
|
import { NoSupportedLinkError } from './NoSupportedLinkError'
|
||||||
|
|
||||||
export const retrieveDomainNodeInfo = async (domain:string):Promise<NodeInfo> => {
|
export const retrieveDomainNodeInfo = async (domain:string):Promise<NodeInfo> => {
|
||||||
const wellKnown = await retrieveWellKnown(domain)
|
const wellKnown = await retrieveWellKnown(domain)
|
||||||
const link = wellKnown.links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0')
|
const link = wellKnown.links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0')
|
||||||
if (typeof link === 'undefined') {
|
if (typeof link === 'undefined') {
|
||||||
throw new Error(`No supported link node info link for ${domain}`)
|
throw new NoSupportedLinkError(domain)
|
||||||
}
|
}
|
||||||
return await retrieveNodeInfo(link.href)
|
return await retrieveNodeInfo(link.href)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,9 @@
|
||||||
import { Provider } from '../Provider'
|
import { Provider } from '../Provider'
|
||||||
import MastodonProvider from '../Mastodon'
|
import MastodonProvider from '../Mastodon'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ecko is Mastodon's fork
|
||||||
|
*/
|
||||||
const EckoProvider: Provider = {
|
const EckoProvider: Provider = {
|
||||||
getKey: () => 'ecko',
|
getKey: () => 'ecko',
|
||||||
getNodeProviders: MastodonProvider.getNodeProviders,
|
getNodeProviders: MastodonProvider.getNodeProviders,
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import { FeedData } from './FeedData'
|
import { FeedProviderMethod } from './FeedProviderMethod'
|
||||||
|
|
||||||
export interface FeedProvider {
|
export interface FeedProvider {
|
||||||
getKey: ()=>string
|
getKey: ()=>string
|
||||||
retrieveFeeds: (domain:string, page:number)=> Promise<FeedData[]>
|
retrieveFeeds: FeedProviderMethod
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,3 @@
|
||||||
|
import { FeedData } from './FeedData'
|
||||||
|
|
||||||
|
export type FeedProviderMethod = (domain: string, page: number) => Promise<FeedData[]>
|
|
@ -1,6 +1,9 @@
|
||||||
import { Provider } from '../Provider'
|
import { Provider } from '../Provider'
|
||||||
import MastodonProvider from '../Mastodon'
|
import MastodonProvider from '../Mastodon'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hometown is Mastodon's fork
|
||||||
|
*/
|
||||||
const HometownProvider: Provider = {
|
const HometownProvider: Provider = {
|
||||||
getKey: () => 'hometown',
|
getKey: () => 'hometown',
|
||||||
getNodeProviders: MastodonProvider.getNodeProviders,
|
getNodeProviders: MastodonProvider.getNodeProviders,
|
||||||
|
|
|
@ -1,8 +1,9 @@
|
||||||
import axios from 'axios'
|
import axios from 'axios'
|
||||||
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
|
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
|
||||||
import { FeedData } from '../FeedData'
|
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
||||||
|
import { FeedProviderMethod } from '../FeedProviderMethod'
|
||||||
|
import { NoMoreFeedsError } from '../NoMoreFeedsError'
|
||||||
|
|
||||||
const limit = 500
|
const limit = 500
|
||||||
|
|
||||||
|
@ -49,49 +50,45 @@ const replaceEmojis = (text: string, emojis: Emoji[]): string => {
|
||||||
return text
|
return text
|
||||||
}
|
}
|
||||||
|
|
||||||
export const retrieveLocalPublicUsersPage = async (domain: string, page: number): Promise<FeedData[]> => {
|
export const retrieveLocalPublicUsersPage: FeedProviderMethod = async (domain, page) => {
|
||||||
try {
|
const response = await axios.get('https://' + domain + '/api/v1/directory', {
|
||||||
const response = await axios.get('https://' + domain + '/api/v1/directory', {
|
params: {
|
||||||
params: {
|
limit: limit,
|
||||||
limit: limit,
|
offset: page * limit,
|
||||||
offset: page * limit,
|
local: true
|
||||||
local: true
|
},
|
||||||
},
|
timeout: getDefaultTimeoutMilliseconds()
|
||||||
timeout: getDefaultTimeoutMilliseconds()
|
})
|
||||||
})
|
assertSuccessJsonResponse(response)
|
||||||
assertSuccessJsonResponse(response)
|
const responseData = schema.parse(response.data)
|
||||||
const responseData = schema.parse(response.data)
|
if (responseData.length === 0) {
|
||||||
if (responseData.length === 0) {
|
throw new NoMoreFeedsError('user')
|
||||||
throw new Error('No more users')
|
|
||||||
}
|
|
||||||
return responseData.map(
|
|
||||||
item => {
|
|
||||||
return {
|
|
||||||
name: item.username,
|
|
||||||
displayName: replaceEmojis(item.display_name, item.emojis),
|
|
||||||
description: replaceEmojis(item.note, item.emojis),
|
|
||||||
followersCount: item.followers_count,
|
|
||||||
followingCount: item.following_count,
|
|
||||||
statusesCount: item.statuses_count,
|
|
||||||
bot: item.bot,
|
|
||||||
url: item.url,
|
|
||||||
avatar: item.avatar,
|
|
||||||
locked: item.locked,
|
|
||||||
lastStatusAt: item.last_status_at !== null ? new Date(item.last_status_at) : null,
|
|
||||||
createdAt: new Date(item.created_at),
|
|
||||||
fields: item.fields.map(field => {
|
|
||||||
return {
|
|
||||||
name: replaceEmojis(field.name, item.emojis),
|
|
||||||
value: replaceEmojis(field.value, item.emojis),
|
|
||||||
verifiedAt: field.verified_at !== null ? new Date(field.verified_at) : null
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
type: 'account',
|
|
||||||
parentFeed: null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error('Invalid response: ' + error)
|
|
||||||
}
|
}
|
||||||
|
return responseData.map(
|
||||||
|
item => {
|
||||||
|
return {
|
||||||
|
name: item.username,
|
||||||
|
displayName: replaceEmojis(item.display_name, item.emojis),
|
||||||
|
description: replaceEmojis(item.note, item.emojis),
|
||||||
|
followersCount: item.followers_count,
|
||||||
|
followingCount: item.following_count,
|
||||||
|
statusesCount: item.statuses_count,
|
||||||
|
bot: item.bot,
|
||||||
|
url: item.url,
|
||||||
|
avatar: item.avatar,
|
||||||
|
locked: item.locked,
|
||||||
|
lastStatusAt: item.last_status_at !== null ? new Date(item.last_status_at) : null,
|
||||||
|
createdAt: new Date(item.created_at),
|
||||||
|
fields: item.fields.map(field => {
|
||||||
|
return {
|
||||||
|
name: replaceEmojis(field.name, item.emojis),
|
||||||
|
value: replaceEmojis(field.value, item.emojis),
|
||||||
|
verifiedAt: field.verified_at !== null ? new Date(field.verified_at) : null
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
type: 'account',
|
||||||
|
parentFeed: null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,22 +2,20 @@ import axios from 'axios'
|
||||||
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
|
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
||||||
|
import { NodeProviderMethod } from '../NodeProviderMethod'
|
||||||
|
import { NoMoreNodesError } from '../NoMoreNodesError'
|
||||||
|
|
||||||
const schema = z.array(
|
const schema = z.array(
|
||||||
z.string()
|
z.string()
|
||||||
)
|
)
|
||||||
|
|
||||||
export const retrievePeers = async (domain:string, page:number):Promise<string[]> => {
|
export const retrievePeers:NodeProviderMethod = async (domain, page) => {
|
||||||
if (page !== 0) {
|
if (page !== 0) {
|
||||||
throw new Error('No more peer pages')
|
throw new NoMoreNodesError('peer')
|
||||||
}
|
|
||||||
try {
|
|
||||||
const response = await axios.get('https://' + domain + '/api/v1/instance/peers', {
|
|
||||||
timeout: getDefaultTimeoutMilliseconds()
|
|
||||||
})
|
|
||||||
assertSuccessJsonResponse(response)
|
|
||||||
return schema.parse(response.data)
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error('Invalid response')
|
|
||||||
}
|
}
|
||||||
|
const response = await axios.get('https://' + domain + '/api/v1/instance/peers', {
|
||||||
|
timeout: getDefaultTimeoutMilliseconds()
|
||||||
|
})
|
||||||
|
assertSuccessJsonResponse(response)
|
||||||
|
return schema.parse(response.data)
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,8 @@ import axios from 'axios'
|
||||||
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
|
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
||||||
|
import { NodeProviderMethod } from '../NodeProviderMethod'
|
||||||
|
import { NoMoreNodesError } from '../NoMoreNodesError'
|
||||||
|
|
||||||
const limit = 100
|
const limit = 100
|
||||||
|
|
||||||
|
@ -11,33 +13,29 @@ const schema = z.array(
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
|
|
||||||
export const retrieveInstancesPage = async (domain: string, page: number): Promise<string[]> => {
|
export const retrieveInstancesPage:NodeProviderMethod = async (domain, page) => {
|
||||||
try {
|
const response = await axios.post('https://' + domain + '/api/federation/instances', {
|
||||||
const response = await axios.post('https://' + domain + '/api/federation/instances', {
|
host: null,
|
||||||
host: null,
|
blocked: null,
|
||||||
blocked: null,
|
notResponding: null,
|
||||||
notResponding: null,
|
suspended: null,
|
||||||
suspended: null,
|
federating: null,
|
||||||
federating: null,
|
subscribing: null,
|
||||||
subscribing: null,
|
publishing: null,
|
||||||
publishing: null,
|
limit: limit,
|
||||||
limit: limit,
|
offset: page * limit,
|
||||||
offset: page * limit,
|
sort: '+id'
|
||||||
sort: '+id'
|
}, {
|
||||||
}, {
|
timeout: getDefaultTimeoutMilliseconds()
|
||||||
timeout: getDefaultTimeoutMilliseconds()
|
})
|
||||||
})
|
assertSuccessJsonResponse(response)
|
||||||
assertSuccessJsonResponse(response)
|
const responseData = schema.parse(response.data)
|
||||||
const responseData = schema.parse(response.data)
|
if (responseData.length === 0) {
|
||||||
if (responseData.length === 0) {
|
throw new NoMoreNodesError('instance')
|
||||||
throw new Error('No more instances')
|
|
||||||
}
|
|
||||||
return responseData.map(
|
|
||||||
item => {
|
|
||||||
return item.host
|
|
||||||
}
|
|
||||||
)
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error('Invalid response: ' + error)
|
|
||||||
}
|
}
|
||||||
|
return responseData.map(
|
||||||
|
item => {
|
||||||
|
return item.host
|
||||||
|
}
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,9 @@ import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
|
||||||
import { FeedData } from '../FeedData'
|
import { FeedData } from '../FeedData'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
||||||
|
import { NodeProviderMethod } from '../NodeProviderMethod'
|
||||||
|
import { NoMoreFeedsError } from '../NoMoreFeedsError'
|
||||||
|
import { FeedProviderMethod } from '../FeedProviderMethod'
|
||||||
|
|
||||||
const limit = 100
|
const limit = 100
|
||||||
|
|
||||||
|
@ -60,57 +63,53 @@ const parseDescription = (description:string|null):string => {
|
||||||
}).join('\n')
|
}).join('\n')
|
||||||
}
|
}
|
||||||
|
|
||||||
export const retrieveUsersPage = async (domain: string, page: number): Promise<FeedData[]> => {
|
export const retrieveUsersPage:FeedProviderMethod = async (domain, page) => {
|
||||||
try {
|
const response = await axios.post('https://' + domain + '/api/users', {
|
||||||
const response = await axios.post('https://' + domain + '/api/users', {
|
state: 'all',
|
||||||
state: 'all',
|
origin: 'local',
|
||||||
origin: 'local',
|
sort: '+createdAt',
|
||||||
sort: '+createdAt',
|
limit: limit,
|
||||||
limit: limit,
|
offset: limit * page
|
||||||
offset: limit * page
|
}, {
|
||||||
}, {
|
timeout: getDefaultTimeoutMilliseconds()
|
||||||
timeout: getDefaultTimeoutMilliseconds()
|
})
|
||||||
})
|
assertSuccessJsonResponse(response)
|
||||||
assertSuccessJsonResponse(response)
|
const responseData = schema.parse(response.data)
|
||||||
const responseData = schema.parse(response.data)
|
if (responseData.length === 0) {
|
||||||
if (responseData.length === 0) {
|
throw new NoMoreFeedsError('user')
|
||||||
throw new Error('No more users')
|
|
||||||
}
|
|
||||||
return responseData.map(
|
|
||||||
item => {
|
|
||||||
return {
|
|
||||||
name: item.username,
|
|
||||||
displayName: replaceEmojis(item.name ?? item.username, item.emojis),
|
|
||||||
description: replaceEmojis(parseDescription(item.description ?? ''), item.emojis),
|
|
||||||
followersCount: item.followersCount,
|
|
||||||
followingCount: item.followingCount,
|
|
||||||
statusesCount: item.notesCount,
|
|
||||||
bot: item.isBot,
|
|
||||||
url: `https://${domain}/@${item.username}`,
|
|
||||||
avatar: item.avatarUrl,
|
|
||||||
locked: item.isLocked,
|
|
||||||
lastStatusAt: item.updatedAt !== null ? new Date(item.updatedAt) : null,
|
|
||||||
createdAt: new Date(item.createdAt),
|
|
||||||
fields: [
|
|
||||||
...item.fields.map(field => {
|
|
||||||
return {
|
|
||||||
name: replaceEmojis(field.name, item.emojis),
|
|
||||||
value: replaceEmojis(field.value, item.emojis),
|
|
||||||
verifiedAt: null
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
...[
|
|
||||||
{ name: 'Location', value: item.location, verifiedAt: null },
|
|
||||||
{ name: 'Birthday', value: item.birthday, verifiedAt: null },
|
|
||||||
{ name: 'Language', value: item.lang, verifiedAt: null }
|
|
||||||
].filter(field => field.value !== null)
|
|
||||||
],
|
|
||||||
type: 'account',
|
|
||||||
parentFeed: null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error('Invalid response: ' + error)
|
|
||||||
}
|
}
|
||||||
|
return responseData.map(
|
||||||
|
item => {
|
||||||
|
return {
|
||||||
|
name: item.username,
|
||||||
|
displayName: replaceEmojis(item.name ?? item.username, item.emojis),
|
||||||
|
description: replaceEmojis(parseDescription(item.description ?? ''), item.emojis),
|
||||||
|
followersCount: item.followersCount,
|
||||||
|
followingCount: item.followingCount,
|
||||||
|
statusesCount: item.notesCount,
|
||||||
|
bot: item.isBot,
|
||||||
|
url: `https://${domain}/@${item.username}`,
|
||||||
|
avatar: item.avatarUrl,
|
||||||
|
locked: item.isLocked,
|
||||||
|
lastStatusAt: item.updatedAt !== null ? new Date(item.updatedAt) : null,
|
||||||
|
createdAt: new Date(item.createdAt),
|
||||||
|
fields: [
|
||||||
|
...item.fields.map(field => {
|
||||||
|
return {
|
||||||
|
name: replaceEmojis(field.name, item.emojis),
|
||||||
|
value: replaceEmojis(field.value, item.emojis),
|
||||||
|
verifiedAt: null
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
...[
|
||||||
|
{ name: 'Location', value: item.location, verifiedAt: null },
|
||||||
|
{ name: 'Birthday', value: item.birthday, verifiedAt: null },
|
||||||
|
{ name: 'Language', value: item.lang, verifiedAt: null }
|
||||||
|
].filter(field => field.value !== null)
|
||||||
|
],
|
||||||
|
type: 'account',
|
||||||
|
parentFeed: null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
export class NoMoreFeedsError extends Error {
|
||||||
|
public constructor (feedType:string) {
|
||||||
|
super(`No more feeds of type ${feedType}`)
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,5 @@
|
||||||
|
export class NoMoreNodesError extends Error {
|
||||||
|
public constructor (nodeType:string) {
|
||||||
|
super(`No more nodes of type ${nodeType}`)
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,4 +1,6 @@
|
||||||
|
import { NodeProviderMethod } from './NodeProviderMethod'
|
||||||
|
|
||||||
export interface NodeProvider {
|
export interface NodeProvider {
|
||||||
getKey:()=>string,
|
getKey:()=>string,
|
||||||
retrieveNodes: (domain: string, page:number)=> Promise<string[]>
|
retrieveNodes: NodeProviderMethod
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,2 @@
|
||||||
|
|
||||||
|
export type NodeProviderMethod = (domain: string, page:number)=> Promise<string[]>
|
|
@ -6,6 +6,8 @@ import { avatarSchema } from './Avatar'
|
||||||
import { parseAvatarUrl } from './parseAvatarUrl'
|
import { parseAvatarUrl } from './parseAvatarUrl'
|
||||||
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
||||||
import { parseDescription } from './parseDescription'
|
import { parseDescription } from './parseDescription'
|
||||||
|
import { NoMoreFeedsError } from '../NoMoreFeedsError'
|
||||||
|
import { FeedProviderMethod } from '../FeedProviderMethod'
|
||||||
|
|
||||||
const limit = 100
|
const limit = 100
|
||||||
|
|
||||||
|
@ -27,7 +29,7 @@ const schema = z.object({
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
export const retrieveAccounts = async (domain: string, page: number): Promise<FeedData[]> => {
|
export const retrieveAccounts:FeedProviderMethod = async (domain, page) => {
|
||||||
const response = await axios.get(`https://${domain}/api/v1/accounts`, {
|
const response = await axios.get(`https://${domain}/api/v1/accounts`, {
|
||||||
params: {
|
params: {
|
||||||
count: limit,
|
count: limit,
|
||||||
|
@ -39,7 +41,7 @@ export const retrieveAccounts = async (domain: string, page: number): Promise<Fe
|
||||||
assertSuccessJsonResponse(response)
|
assertSuccessJsonResponse(response)
|
||||||
const responseData = schema.parse(response.data)
|
const responseData = schema.parse(response.data)
|
||||||
if (responseData.data.length === 0) {
|
if (responseData.data.length === 0) {
|
||||||
throw new Error('No more accounts')
|
throw new NoMoreFeedsError('account')
|
||||||
}
|
}
|
||||||
return responseData.data
|
return responseData.data
|
||||||
.filter(item => item.host === domain)
|
.filter(item => item.host === domain)
|
||||||
|
|
|
@ -2,6 +2,8 @@ import axios from 'axios'
|
||||||
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
|
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
||||||
|
import { NodeProviderMethod } from '../NodeProviderMethod'
|
||||||
|
import { NoMoreNodesError } from '../NoMoreNodesError'
|
||||||
|
|
||||||
const limit = 100
|
const limit = 100
|
||||||
|
|
||||||
|
@ -19,7 +21,7 @@ const schema = z.object({
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
export const retrieveFollowers = async (domain: string, page: number): Promise<string[]> => {
|
export const retrieveFollowers:NodeProviderMethod = async (domain, page) => {
|
||||||
const response = await axios.get(`https://${domain}/api/v1/server/followers`, {
|
const response = await axios.get(`https://${domain}/api/v1/server/followers`, {
|
||||||
params: {
|
params: {
|
||||||
count: limit,
|
count: limit,
|
||||||
|
@ -36,7 +38,7 @@ export const retrieveFollowers = async (domain: string, page: number): Promise<s
|
||||||
hosts.add(item.following.host)
|
hosts.add(item.following.host)
|
||||||
})
|
})
|
||||||
if (hosts.size === 0) {
|
if (hosts.size === 0) {
|
||||||
throw new Error('No more followers')
|
throw new NoMoreNodesError('follower')
|
||||||
}
|
}
|
||||||
return Array.from(hosts)
|
return Array.from(hosts)
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,8 @@ import { avatarSchema } from './Avatar'
|
||||||
import { parseAvatarUrl } from './parseAvatarUrl'
|
import { parseAvatarUrl } from './parseAvatarUrl'
|
||||||
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
||||||
import { parseDescription } from './parseDescription'
|
import { parseDescription } from './parseDescription'
|
||||||
|
import { FeedProviderMethod } from '../FeedProviderMethod'
|
||||||
|
import { NoMoreFeedsError } from '../NoMoreFeedsError'
|
||||||
|
|
||||||
const limit = 100
|
const limit = 100
|
||||||
|
|
||||||
|
@ -34,7 +36,7 @@ const schema = z.object({
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
export const retrieveVideoChannels = async (domain: string, page: number): Promise<FeedData[]> => {
|
export const retrieveVideoChannels:FeedProviderMethod = async (domain, page) => {
|
||||||
const response = await axios.get(`https://${domain}/api/v1/video-channels`, {
|
const response = await axios.get(`https://${domain}/api/v1/video-channels`, {
|
||||||
params: {
|
params: {
|
||||||
count: limit,
|
count: limit,
|
||||||
|
@ -46,7 +48,7 @@ export const retrieveVideoChannels = async (domain: string, page: number): Promi
|
||||||
assertSuccessJsonResponse(response)
|
assertSuccessJsonResponse(response)
|
||||||
const responseData = schema.parse(response.data)
|
const responseData = schema.parse(response.data)
|
||||||
if (responseData.data.length === 0) {
|
if (responseData.data.length === 0) {
|
||||||
throw new Error('No more channels')
|
throw new NoMoreFeedsError('channel')
|
||||||
}
|
}
|
||||||
return responseData.data
|
return responseData.data
|
||||||
.filter(item => item.host === domain)
|
.filter(item => item.host === domain)
|
||||||
|
|
|
@ -1,19 +1,13 @@
|
||||||
import { Provider } from '../Provider'
|
import { Provider } from '../Provider'
|
||||||
import { retrievePeers } from './retrievePeers'
|
import MastodonProvider from '../Mastodon'
|
||||||
import { retrieveLocalPublicUsersPage } from './retrieveLocalPublicUsersPage'
|
|
||||||
import { NodeProvider } from '../NodeProvider'
|
|
||||||
import { FeedProvider } from '../FeedProvider'
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pleroma implements Mastodon's api
|
||||||
|
*/
|
||||||
const PleromaProvider: Provider = {
|
const PleromaProvider: Provider = {
|
||||||
getKey: () => 'pleroma',
|
getKey: () => 'pleroma',
|
||||||
getNodeProviders: ():NodeProvider[] => [{
|
getNodeProviders: MastodonProvider.getNodeProviders,
|
||||||
getKey: () => 'peers',
|
getFeedProviders: MastodonProvider.getFeedProviders
|
||||||
retrieveNodes: retrievePeers
|
|
||||||
}],
|
|
||||||
getFeedProviders: ():FeedProvider[] => [{
|
|
||||||
getKey: () => 'users',
|
|
||||||
retrieveFeeds: retrieveLocalPublicUsersPage
|
|
||||||
}]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export default PleromaProvider
|
export default PleromaProvider
|
||||||
|
|
|
@ -1,100 +0,0 @@
|
||||||
import axios from 'axios'
|
|
||||||
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
|
|
||||||
import { FeedData } from '../FeedData'
|
|
||||||
import { z } from 'zod'
|
|
||||||
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
|
||||||
|
|
||||||
const limit = 500
|
|
||||||
|
|
||||||
const emojiSchema = z.object({
|
|
||||||
shortcode: z.string(),
|
|
||||||
url: z.string()
|
|
||||||
})
|
|
||||||
|
|
||||||
const schema = z.array(
|
|
||||||
z.object({
|
|
||||||
id: z.string(),
|
|
||||||
username: z.string(),
|
|
||||||
display_name: z.string(),
|
|
||||||
locked: z.boolean(),
|
|
||||||
bot: z.boolean(),
|
|
||||||
created_at: z.string(),
|
|
||||||
note: z.string(),
|
|
||||||
url: z.string(),
|
|
||||||
avatar: z.string(),
|
|
||||||
followers_count: z.number(),
|
|
||||||
following_count: z.number(),
|
|
||||||
statuses_count: z.number(),
|
|
||||||
last_status_at: z.nullable(z.string()),
|
|
||||||
emojis: z.array(emojiSchema),
|
|
||||||
fields: z.array(
|
|
||||||
z.object({
|
|
||||||
name: z.string(),
|
|
||||||
value: z.string()
|
|
||||||
})
|
|
||||||
),
|
|
||||||
pleroma: z.object({
|
|
||||||
hide_followers_count: z.boolean(),
|
|
||||||
hide_follows_count: z.boolean()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
)
|
|
||||||
|
|
||||||
type Emoji = z.infer<typeof emojiSchema>
|
|
||||||
|
|
||||||
const replaceEmojis = (text: string, emojis: Emoji[]): string => {
|
|
||||||
emojis.forEach(emoji => {
|
|
||||||
text = text.replace(
|
|
||||||
RegExp(`:${emoji.shortcode}:`, 'gi'),
|
|
||||||
`<img draggable="false" class="emoji" title="${emoji.shortcode}" alt="${emoji.shortcode}" src="${emoji.url}" />`
|
|
||||||
)
|
|
||||||
})
|
|
||||||
return text
|
|
||||||
}
|
|
||||||
|
|
||||||
export const retrieveLocalPublicUsersPage = async (domain: string, page: number): Promise<FeedData[]> => {
|
|
||||||
try {
|
|
||||||
const response = await axios.get('https://' + domain + '/api/v1/directory', {
|
|
||||||
params: {
|
|
||||||
limit: limit,
|
|
||||||
offset: page * limit,
|
|
||||||
local: true
|
|
||||||
},
|
|
||||||
timeout: getDefaultTimeoutMilliseconds()
|
|
||||||
})
|
|
||||||
assertSuccessJsonResponse(response)
|
|
||||||
const responseData = schema.parse(response.data)
|
|
||||||
if (responseData.length === 0) {
|
|
||||||
throw new Error('No more users')
|
|
||||||
}
|
|
||||||
return responseData.map(
|
|
||||||
item => {
|
|
||||||
return {
|
|
||||||
name: item.username,
|
|
||||||
displayName: replaceEmojis(item.display_name, item.emojis),
|
|
||||||
description: replaceEmojis(item.note, item.emojis),
|
|
||||||
followersCount: item.pleroma.hide_followers_count ? null : item.followers_count,
|
|
||||||
followingCount: item.pleroma.hide_follows_count ? null : item.following_count,
|
|
||||||
statusesCount: item.statuses_count,
|
|
||||||
bot: item.bot,
|
|
||||||
url: item.url,
|
|
||||||
avatar: item.avatar,
|
|
||||||
locked: item.locked,
|
|
||||||
lastStatusAt: item.last_status_at !== null ? new Date(item.last_status_at) : null,
|
|
||||||
createdAt: new Date(item.created_at),
|
|
||||||
fields: item.fields.map(field => {
|
|
||||||
return {
|
|
||||||
name: replaceEmojis(field.name, item.emojis),
|
|
||||||
value: replaceEmojis(field.value, item.emojis),
|
|
||||||
verifiedAt: null
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
type: 'account',
|
|
||||||
parentFeed: null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error('Invalid response: ' + error)
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,23 +0,0 @@
|
||||||
import axios from 'axios'
|
|
||||||
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
|
|
||||||
import { z } from 'zod'
|
|
||||||
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
|
||||||
|
|
||||||
const schema = z.array(
|
|
||||||
z.string()
|
|
||||||
)
|
|
||||||
|
|
||||||
export const retrievePeers = async (domain:string, page:number):Promise<string[]> => {
|
|
||||||
if (page !== 0) {
|
|
||||||
throw new Error('No more peer pages')
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const response = await axios.get('https://' + domain + '/api/v1/instance/peers', {
|
|
||||||
timeout: getDefaultTimeoutMilliseconds()
|
|
||||||
})
|
|
||||||
assertSuccessJsonResponse(response)
|
|
||||||
return schema.parse(response.data)
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error('Invalid response')
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -0,0 +1,12 @@
|
||||||
|
export class ProviderKeyAlreadyRegisteredError extends Error {
|
||||||
|
private readonly _key:string
|
||||||
|
|
||||||
|
public constructor (key:string) {
|
||||||
|
super(`Provider with the key ${key} is already registered`)
|
||||||
|
this._key = key
|
||||||
|
}
|
||||||
|
|
||||||
|
public get key (): string {
|
||||||
|
return this._key
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,5 +1,6 @@
|
||||||
import { Provider } from './Provider'
|
import { Provider } from './Provider'
|
||||||
import { Dictionary } from 'typescript-collections'
|
import { Dictionary } from 'typescript-collections'
|
||||||
|
import { ProviderKeyAlreadyRegisteredError } from './ProviderKeyAlreadyRegisteredError'
|
||||||
|
|
||||||
export interface ProviderCallback {
|
export interface ProviderCallback {
|
||||||
(key: string, provider: Provider): void
|
(key: string, provider: Provider): void
|
||||||
|
@ -10,7 +11,7 @@ const providers: Dictionary<string, Provider> = new Dictionary<string, Provider>
|
||||||
const registerProvider = (provider: Provider): void => {
|
const registerProvider = (provider: Provider): void => {
|
||||||
const key = provider.getKey()
|
const key = provider.getKey()
|
||||||
if (providers.containsKey(key)) {
|
if (providers.containsKey(key)) {
|
||||||
throw new Error(`Provider with the key ${key} is already registered`)
|
throw new ProviderKeyAlreadyRegisteredError(key)
|
||||||
}
|
}
|
||||||
providers.setValue(key, provider)
|
providers.setValue(key, provider)
|
||||||
console.info('Added provider to registry', { key: key })
|
console.info('Added provider to registry', { key: key })
|
||||||
|
|
|
@ -0,0 +1,20 @@
|
||||||
|
import { UnexpectedResponseError } from './UnexpectedResponseError'
|
||||||
|
|
||||||
|
export class UnexpectedContentTypeError extends UnexpectedResponseError {
|
||||||
|
private readonly _expectedContentType: string
|
||||||
|
private readonly _actualContentType: string
|
||||||
|
|
||||||
|
public constructor (actualContentType: string, expectedContentType:string) {
|
||||||
|
super(`Expected content type '${expectedContentType}' but got '${actualContentType}'`)
|
||||||
|
this._expectedContentType = expectedContentType
|
||||||
|
this._actualContentType = actualContentType
|
||||||
|
}
|
||||||
|
|
||||||
|
get expectedContentType (): string {
|
||||||
|
return this._expectedContentType
|
||||||
|
}
|
||||||
|
|
||||||
|
get actualContentType (): string {
|
||||||
|
return this._actualContentType
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,3 @@
|
||||||
|
export class UnexpectedResponseError extends Error {
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,20 @@
|
||||||
|
import { UnexpectedResponseError } from './UnexpectedResponseError'
|
||||||
|
|
||||||
|
export class UnexpectedResponseStatusError extends UnexpectedResponseError {
|
||||||
|
private readonly _expectedStatusCode: number
|
||||||
|
private readonly _actualStatusCode: number
|
||||||
|
|
||||||
|
public constructor (expectedStatusCode:number, actualStatusCode:number) {
|
||||||
|
super(`Expected response code ${expectedStatusCode} but got ${actualStatusCode}`)
|
||||||
|
this._actualStatusCode = actualStatusCode
|
||||||
|
this._expectedStatusCode = expectedStatusCode
|
||||||
|
}
|
||||||
|
|
||||||
|
get expectedStatusCode (): number {
|
||||||
|
return this._expectedStatusCode
|
||||||
|
}
|
||||||
|
|
||||||
|
get actualStatusCode (): number {
|
||||||
|
return this._actualStatusCode
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,10 +1,16 @@
|
||||||
import { AxiosResponse } from 'axios'
|
import { AxiosResponse } from 'axios'
|
||||||
|
import { UnexpectedResponseStatusError } from './UnexpectedResponseStatusError'
|
||||||
|
import { UnexpectedContentTypeError } from './UnexpectedContentTypeError'
|
||||||
|
|
||||||
export const assertSuccessJsonResponse = (response:AxiosResponse<unknown>):void => {
|
export const assertSuccessJsonResponse = (response: AxiosResponse<unknown>): void => {
|
||||||
if (response.status !== 200) {
|
const expectedStatus = 200
|
||||||
throw new Error('Unexpected response ' + response.status)
|
const actualStatus = response.status
|
||||||
|
if (actualStatus !== expectedStatus) {
|
||||||
|
throw new UnexpectedResponseStatusError(expectedStatus, actualStatus)
|
||||||
}
|
}
|
||||||
if (!response.headers['content-type'].startsWith('application/json')) {
|
const expectedContentType = 'application/json'
|
||||||
throw new Error('Unexpected content-type ' + response.headers['content-type'])
|
const actualContentType = response.headers['content-type']
|
||||||
|
if (!actualContentType.startsWith(expectedContentType)) {
|
||||||
|
throw new UnexpectedContentTypeError(expectedContentType, actualContentType)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,6 +8,7 @@ import { findNewNodes } from './Nodes/findNewNodes'
|
||||||
import { NodeProvider } from '../Fediverse/Providers/NodeProvider'
|
import { NodeProvider } from '../Fediverse/Providers/NodeProvider'
|
||||||
import { FeedProvider } from '../Fediverse/Providers/FeedProvider'
|
import { FeedProvider } from '../Fediverse/Providers/FeedProvider'
|
||||||
import { refreshFeeds } from './Feeds/refreshFeeds'
|
import { refreshFeeds } from './Feeds/refreshFeeds'
|
||||||
|
import { deleteOldFeeds } from '../Storage/Feeds/deleteOldFeeds'
|
||||||
|
|
||||||
export const processNextNode = async (prisma:PrismaClient, providerRegistry:ProviderRegistry):Promise<void> => {
|
export const processNextNode = async (prisma:PrismaClient, providerRegistry:ProviderRegistry):Promise<void> => {
|
||||||
console.info('#############################################')
|
console.info('#############################################')
|
||||||
|
@ -18,6 +19,7 @@ export const processNextNode = async (prisma:PrismaClient, providerRegistry:Prov
|
||||||
|
|
||||||
if (!providerRegistry.containsKey(node.softwareName)) {
|
if (!providerRegistry.containsKey(node.softwareName)) {
|
||||||
console.warn('Unknown software', { domain: node.domain, software: node.softwareName })
|
console.warn('Unknown software', { domain: node.domain, software: node.softwareName })
|
||||||
|
await deleteOldFeeds(prisma, node)
|
||||||
await setNodeRefreshed(prisma, node)
|
await setNodeRefreshed(prisma, node)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -37,5 +39,7 @@ export const processNextNode = async (prisma:PrismaClient, providerRegistry:Prov
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
|
|
||||||
|
await deleteOldFeeds(prisma, node)
|
||||||
|
|
||||||
await setNodeRefreshed(prisma, node)
|
await setNodeRefreshed(prisma, node)
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,18 @@
|
||||||
|
import { Node, PrismaClient } from '@prisma/client'
|
||||||
|
|
||||||
|
export const deleteOldFeeds = async (prisma: PrismaClient, node: Node): Promise<number> => {
|
||||||
|
const result = await prisma.feed.deleteMany({
|
||||||
|
where: {
|
||||||
|
nodeId: {
|
||||||
|
equals: node.id
|
||||||
|
},
|
||||||
|
refreshedAt: {
|
||||||
|
lt: node.refreshAttemptedAt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
console.info('Deleted old feeds', {
|
||||||
|
count: result.count, olderThen: node.refreshAttemptedAt, nodeDomain: node.domain
|
||||||
|
})
|
||||||
|
return result.count
|
||||||
|
}
|
|
@ -0,0 +1,5 @@
|
||||||
|
export class NoNodeFoundError extends Error {
|
||||||
|
public constructor () {
|
||||||
|
super('No node found')
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,4 +1,5 @@
|
||||||
import { Node, PrismaClient } from '@prisma/client'
|
import { Node, PrismaClient } from '@prisma/client'
|
||||||
|
import { NoNodeFoundError } from './NoNodeFoundError'
|
||||||
|
|
||||||
export const fetchNodeToProcess = async (prisma: PrismaClient): Promise<Node> => {
|
export const fetchNodeToProcess = async (prisma: PrismaClient): Promise<Node> => {
|
||||||
const currentTimestamp = Date.now()
|
const currentTimestamp = Date.now()
|
||||||
|
@ -56,10 +57,10 @@ export const fetchNodeToProcess = async (prisma: PrismaClient): Promise<Node> =>
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
if (node) {
|
if (!node) {
|
||||||
console.log('Found oldest node', { domain: node.domain })
|
throw new NoNodeFoundError()
|
||||||
} else {
|
|
||||||
throw new Error('No node found')
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
console.log('Found oldest node', { domain: node.domain })
|
||||||
return node
|
return node
|
||||||
}
|
}
|
||||||
|
|
Ładowanie…
Reference in New Issue