mirror
/
fedicrawl
kopia lustrzana https://github.com/Stopka/fedicrawl
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Not existing feeds are deleted from databse

main
Štěpán Škorpil 2022-04-17 14:03:53 +02:00
rodzic cefbd76066
commit 3ed37c04ce
34 zmienionych plików z 497 dodań i 313 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

23
application/package-lock.json wygenerowano
Wyświetl plik

@ -3469,9 +3469,9 @@
"dev": true
},
"node_modules/follow-redirects": {
"version": "1.14.5",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz",
"integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==",
"version": "1.14.9",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
"integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==",
"funding": [
{
"type": "individual",
@ -6344,9 +6344,10 @@
}
},
"node_modules/minimist": {
"version": "1.2.5",
"dev": true,
"license": "MIT"
"version": "1.2.6",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
"integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
"dev": true
},
"node_modules/ms": {
"version": "2.1.2",
@ -11163,9 +11164,9 @@
"dev": true
},
"follow-redirects": {
"version": "1.14.5",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz",
"integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA=="
"version": "1.14.9",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
"integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w=="
},
"form-data": {
"version": "3.0.1",
@ -13249,7 +13250,9 @@
}
},
"minimist": {
"version": "1.2.5",
"version": "1.2.6",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
"integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
"dev": true
},
"ms": {

12
application/package.json
Wyświetl plik

@ -26,16 +26,16 @@
"migrate:resolve": "npx prisma migrate resolve",
"prisma:generate": "npx prisma generate",
"prisma:studio": "npx prisma studio",
"start:deploy": "npm run migrate:deploy && npm run start"
"start:deploy": "npm run migrate:deploy && npm run start"
},
"dependencies": {
"@prisma/client": "^3.6.0",
"axios": "^0.21.1",
"npmlog": "^6.0.0",
"typescript-collections": "^1.3.3",
"zod": "^3.11.6",
"rimraf": "^3.0.2",
"striptags": "^3.2.0"
"striptags": "^3.2.0",
"typescript-collections": "^1.3.3",
"zod": "^3.11.6"
},
"devDependencies": {
"@types/jest": "^27.0.2",
@ -49,11 +49,11 @@
"eslint-plugin-node": "^11.1.0",
"eslint-plugin-promise": "^5.1.1",
"eslint-plugin-react": "^7.27.1",
"jest": "^27.3.0",
"prisma": "^3.6.0",
"standard": "*",
"typescript": "^4.3.5",
"ts-jest": "^27.0.7",
"jest": "^27.3.0"
"typescript": "^4.3.5"
},
"jest": {
"moduleFileExtensions": [

193
application/prisma/migrations/20220417112556_add_delete_cascade/migration.sql 100644
Wyświetl plik

@ -0,0 +1,193 @@
/*
Warnings:
- A unique constraint covering the columns `[name,nodeId]` on the table `Feed` will be added. If there are existing duplicate values, this will fail.
- A unique constraint covering the columns `[domain]` on the table `Node` will be added. If there are existing duplicate values, this will fail.
- A unique constraint covering the columns `[name]` on the table `Tag` will be added. If there are existing duplicate values, this will fail.
*/
-- DropForeignKey
ALTER TABLE "Email" DROP CONSTRAINT "Email_feedId_fkey";
-- DropForeignKey
ALTER TABLE "Feed" DROP CONSTRAINT "Feed_nodeId_fkey";
-- DropForeignKey
ALTER TABLE "FeedToTag" DROP CONSTRAINT "FeedToTag_feedId_fkey";
-- DropForeignKey
ALTER TABLE "FeedToTag" DROP CONSTRAINT "FeedToTag_tagId_fkey";
-- DropForeignKey
ALTER TABLE "Field" DROP CONSTRAINT "Field_feedId_fkey";
-- DropIndex
DROP INDEX "Email_address_idx";
-- DropIndex
DROP INDEX "Feed_bot_idx";
-- DropIndex
DROP INDEX "Feed_createdAt_idx";
-- DropIndex
DROP INDEX "Feed_description_idx";
-- DropIndex
DROP INDEX "Feed_displayName_idx";
-- DropIndex
DROP INDEX "Feed_fulltext_idx";
-- DropIndex
DROP INDEX "Feed_lastStatusAt_idx";
-- DropIndex
DROP INDEX "Feed_locked_idx";
-- DropIndex
DROP INDEX "Feed_name_nodeId_key";
-- DropIndex
DROP INDEX "Feed_parentFeedName_parentFeedDomain_idx";
-- DropIndex
DROP INDEX "Feed_refreshedAt_idx";
-- DropIndex
DROP INDEX "Feed_type_idx";
-- DropIndex
DROP INDEX "Field_name_idx";
-- DropIndex
DROP INDEX "Field_value_idx";
-- DropIndex
DROP INDEX "Node_domain_key";
-- DropIndex
DROP INDEX "Node_foundAt_idx";
-- DropIndex
DROP INDEX "Node_halfYearActiveUserCount_idx";
-- DropIndex
DROP INDEX "Node_monthActiveUserCount_idx";
-- DropIndex
DROP INDEX "Node_openRegistrations_idx";
-- DropIndex
DROP INDEX "Node_refreshAttemptedAt_idx";
-- DropIndex
DROP INDEX "Node_refreshedAt_idx";
-- DropIndex
DROP INDEX "Node_softwareName_idx";
-- DropIndex
DROP INDEX "Node_softwareVersion_idx";
-- DropIndex
DROP INDEX "Node_statusesCount_idx";
-- DropIndex
DROP INDEX "Node_totalUserCount_idx";
-- DropIndex
DROP INDEX "Tag_name_key";
-- CreateIndex
CREATE INDEX "Email_address_idx" ON "Email"("address");
-- CreateIndex
CREATE INDEX "Feed_displayName_idx" ON "Feed"("displayName");
-- CreateIndex
CREATE INDEX "Feed_description_idx" ON "Feed"("description");
-- CreateIndex
CREATE INDEX "Feed_bot_idx" ON "Feed"("bot");
-- CreateIndex
CREATE INDEX "Feed_locked_idx" ON "Feed"("locked");
-- CreateIndex
CREATE INDEX "Feed_lastStatusAt_idx" ON "Feed"("lastStatusAt");
-- CreateIndex
CREATE INDEX "Feed_createdAt_idx" ON "Feed"("createdAt");
-- CreateIndex
CREATE INDEX "Feed_refreshedAt_idx" ON "Feed"("refreshedAt");
-- CreateIndex
CREATE INDEX "Feed_parentFeedName_parentFeedDomain_idx" ON "Feed"("parentFeedName", "parentFeedDomain");
-- CreateIndex
CREATE INDEX "Feed_type_idx" ON "Feed"("type");
-- CreateIndex
CREATE INDEX "Feed_fulltext_idx" ON "Feed"("fulltext");
-- CreateIndex
CREATE UNIQUE INDEX "Feed_name_nodeId_key" ON "Feed"("name", "nodeId");
-- CreateIndex
CREATE INDEX "Field_name_idx" ON "Field"("name");
-- CreateIndex
CREATE INDEX "Field_value_idx" ON "Field"("value");
-- CreateIndex
CREATE UNIQUE INDEX "Node_domain_key" ON "Node"("domain");
-- CreateIndex
CREATE INDEX "Node_softwareName_idx" ON "Node"("softwareName");
-- CreateIndex
CREATE INDEX "Node_softwareVersion_idx" ON "Node"("softwareVersion");
-- CreateIndex
CREATE INDEX "Node_totalUserCount_idx" ON "Node"("totalUserCount");
-- CreateIndex
CREATE INDEX "Node_monthActiveUserCount_idx" ON "Node"("monthActiveUserCount");
-- CreateIndex
CREATE INDEX "Node_halfYearActiveUserCount_idx" ON "Node"("halfYearActiveUserCount");
-- CreateIndex
CREATE INDEX "Node_statusesCount_idx" ON "Node"("statusesCount");
-- CreateIndex
CREATE INDEX "Node_openRegistrations_idx" ON "Node"("openRegistrations");
-- CreateIndex
CREATE INDEX "Node_refreshedAt_idx" ON "Node"("refreshedAt");
-- CreateIndex
CREATE INDEX "Node_refreshAttemptedAt_idx" ON "Node"("refreshAttemptedAt");
-- CreateIndex
CREATE INDEX "Node_foundAt_idx" ON "Node"("foundAt");
-- CreateIndex
CREATE UNIQUE INDEX "Tag_name_key" ON "Tag"("name");
-- AddForeignKey
ALTER TABLE "Email" ADD CONSTRAINT "Email_feedId_fkey" FOREIGN KEY ("feedId") REFERENCES "Feed"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "FeedToTag" ADD CONSTRAINT "FeedToTag_feedId_fkey" FOREIGN KEY ("feedId") REFERENCES "Feed"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "FeedToTag" ADD CONSTRAINT "FeedToTag_tagId_fkey" FOREIGN KEY ("tagId") REFERENCES "Tag"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "Field" ADD CONSTRAINT "Field_feedId_fkey" FOREIGN KEY ("feedId") REFERENCES "Feed"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "Feed" ADD CONSTRAINT "Feed_nodeId_fkey" FOREIGN KEY ("nodeId") REFERENCES "Node"("id") ON DELETE CASCADE ON UPDATE CASCADE;

12
application/prisma/schema.prisma
Wyświetl plik

@ -5,7 +5,7 @@ datasource db {
generator client {
provider = "prisma-client-js"
previewFeatures = ["extendedIndexes","fullTextSearch"]
previewFeatures = ["extendedIndexes","fullTextSearch","referentialActions"]
}
model Tag {
@ -17,16 +17,16 @@ model Tag {
model Email {
id String @id @default(uuid()) @db.Uuid
address String
feed Feed @relation(fields: [feedId], references: [id])
feed Feed @relation(fields: [feedId], references: [id], onDelete: Cascade)
feedId String @db.Uuid
@@index([address])
}
model FeedToTag {
feed Feed @relation(fields: [feedId], references: [id])
feed Feed @relation(fields: [feedId], references: [id], onDelete: Cascade)
feedId String @db.Uuid
tag Tag @relation(fields: [tagId], references: [id])
tag Tag @relation(fields: [tagId], references: [id], onDelete: Cascade)
tagId String @db.Uuid
@@id([feedId, tagId])
@ -36,7 +36,7 @@ model Field {
id String @id @default(uuid()) @db.Uuid
name String
value String
feed Feed @relation(fields: [feedId], references: [id])
feed Feed @relation(fields: [feedId], references: [id], onDelete: Cascade)
feedId String @db.Uuid
@@index([name])
@ -50,7 +50,7 @@ enum FeedType{
model Feed {
id String @id @default(uuid()) @db.Uuid
node Node @relation(fields: [nodeId], references: [id])
node Node @relation(fields: [nodeId], references: [id], onDelete: Cascade)
nodeId String @db.Uuid
foundAt DateTime @default(now())
refreshedAt DateTime @updatedAt

5
application/src/Fediverse/NodeInfo/NoSupportedLinkError.ts 100644
Wyświetl plik

@ -0,0 +1,5 @@
export class NoSupportedLinkError extends Error {
public constructor (domain:string) {
super(`No supported link node info link for ${domain}`)
}
}

3
application/src/Fediverse/NodeInfo/retrieveDomainNodeInfo.ts
Wyświetl plik

@ -1,11 +1,12 @@
import { retrieveWellKnown } from './retrieveWellKnown'
import { retrieveNodeInfo, NodeInfo } from './retrieveNodeInfo'
import { NoSupportedLinkError } from './NoSupportedLinkError'
export const retrieveDomainNodeInfo = async (domain:string):Promise<NodeInfo> => {
const wellKnown = await retrieveWellKnown(domain)
const link = wellKnown.links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0')
if (typeof link === 'undefined') {
throw new Error(`No supported link node info link for ${domain}`)
throw new NoSupportedLinkError(domain)
}
return await retrieveNodeInfo(link.href)
}

3
application/src/Fediverse/Providers/Ecko/index.ts
Wyświetl plik

@ -1,6 +1,9 @@
import { Provider } from '../Provider'
import MastodonProvider from '../Mastodon'
/**
* Ecko is Mastodon's fork
*/
const EckoProvider: Provider = {
getKey: () => 'ecko',
getNodeProviders: MastodonProvider.getNodeProviders,

4
application/src/Fediverse/Providers/FeedProvider.ts
Wyświetl plik

@ -1,6 +1,6 @@
import { FeedData } from './FeedData'
import { FeedProviderMethod } from './FeedProviderMethod'
export interface FeedProvider {
getKey: ()=>string
retrieveFeeds: (domain:string, page:number)=> Promise<FeedData[]>
retrieveFeeds: FeedProviderMethod
}

3
application/src/Fediverse/Providers/FeedProviderMethod.ts 100644
Wyświetl plik

@ -0,0 +1,3 @@
import { FeedData } from './FeedData'
export type FeedProviderMethod = (domain: string, page: number) => Promise<FeedData[]>

3
application/src/Fediverse/Providers/Hometown/index.ts
Wyświetl plik

@ -1,6 +1,9 @@
import { Provider } from '../Provider'
import MastodonProvider from '../Mastodon'
/**
* Hometown is Mastodon's fork
*/
const HometownProvider: Provider = {
getKey: () => 'hometown',
getNodeProviders: MastodonProvider.getNodeProviders,

87
application/src/Fediverse/Providers/Mastodon/retrieveLocalPublicUsersPage.ts
Wyświetl plik

@ -1,8 +1,9 @@
import axios from 'axios'
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
import { FeedData } from '../FeedData'
import { z } from 'zod'
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
import { FeedProviderMethod } from '../FeedProviderMethod'
import { NoMoreFeedsError } from '../NoMoreFeedsError'
const limit = 500
@ -49,49 +50,45 @@ const replaceEmojis = (text: string, emojis: Emoji[]): string => {
return text
}
export const retrieveLocalPublicUsersPage = async (domain: string, page: number): Promise<FeedData[]> => {
try {
const response = await axios.get('https://' + domain + '/api/v1/directory', {
params: {
limit: limit,
offset: page * limit,
local: true
},
timeout: getDefaultTimeoutMilliseconds()
})
assertSuccessJsonResponse(response)
const responseData = schema.parse(response.data)
if (responseData.length === 0) {
throw new Error('No more users')
}
return responseData.map(
item => {
return {
name: item.username,
displayName: replaceEmojis(item.display_name, item.emojis),
description: replaceEmojis(item.note, item.emojis),
followersCount: item.followers_count,
followingCount: item.following_count,
statusesCount: item.statuses_count,
bot: item.bot,
url: item.url,
avatar: item.avatar,
locked: item.locked,
lastStatusAt: item.last_status_at !== null ? new Date(item.last_status_at) : null,
createdAt: new Date(item.created_at),
fields: item.fields.map(field => {
return {
name: replaceEmojis(field.name, item.emojis),
value: replaceEmojis(field.value, item.emojis),
verifiedAt: field.verified_at !== null ? new Date(field.verified_at) : null
}
}),
type: 'account',
parentFeed: null
}
}
)
} catch (error) {
throw new Error('Invalid response: ' + error)
export const retrieveLocalPublicUsersPage: FeedProviderMethod = async (domain, page) => {
const response = await axios.get('https://' + domain + '/api/v1/directory', {
params: {
limit: limit,
offset: page * limit,
local: true
},
timeout: getDefaultTimeoutMilliseconds()
})
assertSuccessJsonResponse(response)
const responseData = schema.parse(response.data)
if (responseData.length === 0) {
throw new NoMoreFeedsError('user')
}
return responseData.map(
item => {
return {
name: item.username,
displayName: replaceEmojis(item.display_name, item.emojis),
description: replaceEmojis(item.note, item.emojis),
followersCount: item.followers_count,
followingCount: item.following_count,
statusesCount: item.statuses_count,
bot: item.bot,
url: item.url,
avatar: item.avatar,
locked: item.locked,
lastStatusAt: item.last_status_at !== null ? new Date(item.last_status_at) : null,
createdAt: new Date(item.created_at),
fields: item.fields.map(field => {
return {
name: replaceEmojis(field.name, item.emojis),
value: replaceEmojis(field.value, item.emojis),
verifiedAt: field.verified_at !== null ? new Date(field.verified_at) : null
}
}),
type: 'account',
parentFeed: null
}
}
)
}

20
application/src/Fediverse/Providers/Mastodon/retrievePeers.ts
Wyświetl plik

@ -2,22 +2,20 @@ import axios from 'axios'
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
import { z } from 'zod'
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
import { NodeProviderMethod } from '../NodeProviderMethod'
import { NoMoreNodesError } from '../NoMoreNodesError'
const schema = z.array(
z.string()
)
export const retrievePeers = async (domain:string, page:number):Promise<string[]> => {
export const retrievePeers:NodeProviderMethod = async (domain, page) => {
if (page !== 0) {
throw new Error('No more peer pages')
}
try {
const response = await axios.get('https://' + domain + '/api/v1/instance/peers', {
timeout: getDefaultTimeoutMilliseconds()
})
assertSuccessJsonResponse(response)
return schema.parse(response.data)
} catch (error) {
throw new Error('Invalid response')
throw new NoMoreNodesError('peer')
}
const response = await axios.get('https://' + domain + '/api/v1/instance/peers', {
timeout: getDefaultTimeoutMilliseconds()
})
assertSuccessJsonResponse(response)
return schema.parse(response.data)
}

54
application/src/Fediverse/Providers/Misskey/retrieveInstancesPage.ts
Wyświetl plik

@ -2,6 +2,8 @@ import axios from 'axios'
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
import { z } from 'zod'
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
import { NodeProviderMethod } from '../NodeProviderMethod'
import { NoMoreNodesError } from '../NoMoreNodesError'
const limit = 100
@ -11,33 +13,29 @@ const schema = z.array(
})
)
export const retrieveInstancesPage = async (domain: string, page: number): Promise<string[]> => {
try {
const response = await axios.post('https://' + domain + '/api/federation/instances', {
host: null,
blocked: null,
notResponding: null,
suspended: null,
federating: null,
subscribing: null,
publishing: null,
limit: limit,
offset: page * limit,
sort: '+id'
}, {
timeout: getDefaultTimeoutMilliseconds()
})
assertSuccessJsonResponse(response)
const responseData = schema.parse(response.data)
if (responseData.length === 0) {
throw new Error('No more instances')
}
return responseData.map(
item => {
return item.host
}
)
} catch (error) {
throw new Error('Invalid response: ' + error)
export const retrieveInstancesPage:NodeProviderMethod = async (domain, page) => {
const response = await axios.post('https://' + domain + '/api/federation/instances', {
host: null,
blocked: null,
notResponding: null,
suspended: null,
federating: null,
subscribing: null,
publishing: null,
limit: limit,
offset: page * limit,
sort: '+id'
}, {
timeout: getDefaultTimeoutMilliseconds()
})
assertSuccessJsonResponse(response)
const responseData = schema.parse(response.data)
if (responseData.length === 0) {
throw new NoMoreNodesError('instance')
}
return responseData.map(
item => {
return item.host
}
)
}

103
application/src/Fediverse/Providers/Misskey/retrieveUsersPage.ts
Wyświetl plik

@ -3,6 +3,9 @@ import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
import { FeedData } from '../FeedData'
import { z } from 'zod'
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
import { NodeProviderMethod } from '../NodeProviderMethod'
import { NoMoreFeedsError } from '../NoMoreFeedsError'
import { FeedProviderMethod } from '../FeedProviderMethod'
const limit = 100
@ -60,57 +63,53 @@ const parseDescription = (description:string|null):string => {
}).join('\n')
}
export const retrieveUsersPage = async (domain: string, page: number): Promise<FeedData[]> => {
try {
const response = await axios.post('https://' + domain + '/api/users', {
state: 'all',
origin: 'local',
sort: '+createdAt',
limit: limit,
offset: limit * page
}, {
timeout: getDefaultTimeoutMilliseconds()
})
assertSuccessJsonResponse(response)
const responseData = schema.parse(response.data)
if (responseData.length === 0) {
throw new Error('No more users')
}
return responseData.map(
item => {
return {
name: item.username,
displayName: replaceEmojis(item.name ?? item.username, item.emojis),
description: replaceEmojis(parseDescription(item.description ?? ''), item.emojis),
followersCount: item.followersCount,
followingCount: item.followingCount,
statusesCount: item.notesCount,
bot: item.isBot,
url: `https://${domain}/@${item.username}`,
avatar: item.avatarUrl,
locked: item.isLocked,
lastStatusAt: item.updatedAt !== null ? new Date(item.updatedAt) : null,
createdAt: new Date(item.createdAt),
fields: [
...item.fields.map(field => {
return {
name: replaceEmojis(field.name, item.emojis),
value: replaceEmojis(field.value, item.emojis),
verifiedAt: null
}
}),
...[
{ name: 'Location', value: item.location, verifiedAt: null },
{ name: 'Birthday', value: item.birthday, verifiedAt: null },
{ name: 'Language', value: item.lang, verifiedAt: null }
].filter(field => field.value !== null)
],
type: 'account',
parentFeed: null
}
}
)
} catch (error) {
throw new Error('Invalid response: ' + error)
export const retrieveUsersPage:FeedProviderMethod = async (domain, page) => {
const response = await axios.post('https://' + domain + '/api/users', {
state: 'all',
origin: 'local',
sort: '+createdAt',
limit: limit,
offset: limit * page
}, {
timeout: getDefaultTimeoutMilliseconds()
})
assertSuccessJsonResponse(response)
const responseData = schema.parse(response.data)
if (responseData.length === 0) {
throw new NoMoreFeedsError('user')
}
return responseData.map(
item => {
return {
name: item.username,
displayName: replaceEmojis(item.name ?? item.username, item.emojis),
description: replaceEmojis(parseDescription(item.description ?? ''), item.emojis),
followersCount: item.followersCount,
followingCount: item.followingCount,
statusesCount: item.notesCount,
bot: item.isBot,
url: `https://${domain}/@${item.username}`,
avatar: item.avatarUrl,
locked: item.isLocked,
lastStatusAt: item.updatedAt !== null ? new Date(item.updatedAt) : null,
createdAt: new Date(item.createdAt),
fields: [
...item.fields.map(field => {
return {
name: replaceEmojis(field.name, item.emojis),
value: replaceEmojis(field.value, item.emojis),
verifiedAt: null
}
}),
...[
{ name: 'Location', value: item.location, verifiedAt: null },
{ name: 'Birthday', value: item.birthday, verifiedAt: null },
{ name: 'Language', value: item.lang, verifiedAt: null }
].filter(field => field.value !== null)
],
type: 'account',
parentFeed: null
}
}
)
}

5
application/src/Fediverse/Providers/NoMoreFeedsError.ts 100644
Wyświetl plik

@ -0,0 +1,5 @@
export class NoMoreFeedsError extends Error {
public constructor (feedType:string) {
super(`No more feeds of type ${feedType}`)
}
}

5
application/src/Fediverse/Providers/NoMoreNodesError.ts 100644
Wyświetl plik

@ -0,0 +1,5 @@
export class NoMoreNodesError extends Error {
public constructor (nodeType:string) {
super(`No more nodes of type ${nodeType}`)
}
}

4
application/src/Fediverse/Providers/NodeProvider.ts
Wyświetl plik

@ -1,4 +1,6 @@
import { NodeProviderMethod } from './NodeProviderMethod'
export interface NodeProvider {
getKey:()=>string,
retrieveNodes: (domain: string, page:number)=> Promise<string[]>
retrieveNodes: NodeProviderMethod
}

2
application/src/Fediverse/Providers/NodeProviderMethod.ts 100644
Wyświetl plik

@ -0,0 +1,2 @@
export type NodeProviderMethod = (domain: string, page:number)=> Promise<string[]>

6
application/src/Fediverse/Providers/Peertube/retrieveAccounts.ts
Wyświetl plik

@ -6,6 +6,8 @@ import { avatarSchema } from './Avatar'
import { parseAvatarUrl } from './parseAvatarUrl'
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
import { parseDescription } from './parseDescription'
import { NoMoreFeedsError } from '../NoMoreFeedsError'
import { FeedProviderMethod } from '../FeedProviderMethod'
const limit = 100
@ -27,7 +29,7 @@ const schema = z.object({
)
})
export const retrieveAccounts = async (domain: string, page: number): Promise<FeedData[]> => {
export const retrieveAccounts:FeedProviderMethod = async (domain, page) => {
const response = await axios.get(`https://${domain}/api/v1/accounts`, {
params: {
count: limit,
@ -39,7 +41,7 @@ export const retrieveAccounts = async (domain: string, page: number): Promise<Fe
assertSuccessJsonResponse(response)
const responseData = schema.parse(response.data)
if (responseData.data.length === 0) {
throw new Error('No more accounts')
throw new NoMoreFeedsError('account')
}
return responseData.data
.filter(item => item.host === domain)

6
application/src/Fediverse/Providers/Peertube/retrieveFollowers.ts
Wyświetl plik

@ -2,6 +2,8 @@ import axios from 'axios'
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
import { z } from 'zod'
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
import { NodeProviderMethod } from '../NodeProviderMethod'
import { NoMoreNodesError } from '../NoMoreNodesError'
const limit = 100
@ -19,7 +21,7 @@ const schema = z.object({
)
})
export const retrieveFollowers = async (domain: string, page: number): Promise<string[]> => {
export const retrieveFollowers:NodeProviderMethod = async (domain, page) => {
const response = await axios.get(`https://${domain}/api/v1/server/followers`, {
params: {
count: limit,
@ -36,7 +38,7 @@ export const retrieveFollowers = async (domain: string, page: number): Promise<s
hosts.add(item.following.host)
})
if (hosts.size === 0) {
throw new Error('No more followers')
throw new NoMoreNodesError('follower')
}
return Array.from(hosts)
}

6
application/src/Fediverse/Providers/Peertube/retrieveVideoChannels.ts
Wyświetl plik

@ -7,6 +7,8 @@ import { avatarSchema } from './Avatar'
import { parseAvatarUrl } from './parseAvatarUrl'
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
import { parseDescription } from './parseDescription'
import { FeedProviderMethod } from '../FeedProviderMethod'
import { NoMoreFeedsError } from '../NoMoreFeedsError'
const limit = 100
@ -34,7 +36,7 @@ const schema = z.object({
)
})
export const retrieveVideoChannels = async (domain: string, page: number): Promise<FeedData[]> => {
export const retrieveVideoChannels:FeedProviderMethod = async (domain, page) => {
const response = await axios.get(`https://${domain}/api/v1/video-channels`, {
params: {
count: limit,
@ -46,7 +48,7 @@ export const retrieveVideoChannels = async (domain: string, page: number): Promi
assertSuccessJsonResponse(response)
const responseData = schema.parse(response.data)
if (responseData.data.length === 0) {
throw new Error('No more channels')
throw new NoMoreFeedsError('channel')
}
return responseData.data
.filter(item => item.host === domain)

18
application/src/Fediverse/Providers/Pleroma/index.ts
Wyświetl plik

@ -1,19 +1,13 @@
import { Provider } from '../Provider'
import { retrievePeers } from './retrievePeers'
import { retrieveLocalPublicUsersPage } from './retrieveLocalPublicUsersPage'
import { NodeProvider } from '../NodeProvider'
import { FeedProvider } from '../FeedProvider'
import MastodonProvider from '../Mastodon'
/**
* Pleroma implements Mastodon's api
*/
const PleromaProvider: Provider = {
getKey: () => 'pleroma',
getNodeProviders: ():NodeProvider[] => [{
getKey: () => 'peers',
retrieveNodes: retrievePeers
}],
getFeedProviders: ():FeedProvider[] => [{
getKey: () => 'users',
retrieveFeeds: retrieveLocalPublicUsersPage
}]
getNodeProviders: MastodonProvider.getNodeProviders,
getFeedProviders: MastodonProvider.getFeedProviders
}
export default PleromaProvider

100
application/src/Fediverse/Providers/Pleroma/retrieveLocalPublicUsersPage.ts
Wyświetl plik

@ -1,100 +0,0 @@
import axios from 'axios'
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
import { FeedData } from '../FeedData'
import { z } from 'zod'
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
const limit = 500
const emojiSchema = z.object({
shortcode: z.string(),
url: z.string()
})
const schema = z.array(
z.object({
id: z.string(),
username: z.string(),
display_name: z.string(),
locked: z.boolean(),
bot: z.boolean(),
created_at: z.string(),
note: z.string(),
url: z.string(),
avatar: z.string(),
followers_count: z.number(),
following_count: z.number(),
statuses_count: z.number(),
last_status_at: z.nullable(z.string()),
emojis: z.array(emojiSchema),
fields: z.array(
z.object({
name: z.string(),
value: z.string()
})
),
pleroma: z.object({
hide_followers_count: z.boolean(),
hide_follows_count: z.boolean()
})
})
)
type Emoji = z.infer<typeof emojiSchema>
const replaceEmojis = (text: string, emojis: Emoji[]): string => {
emojis.forEach(emoji => {
text = text.replace(
RegExp(`:${emoji.shortcode}:`, 'gi'),
`<img draggable="false" class="emoji" title="${emoji.shortcode}" alt="${emoji.shortcode}" src="${emoji.url}" />`
)
})
return text
}
export const retrieveLocalPublicUsersPage = async (domain: string, page: number): Promise<FeedData[]> => {
try {
const response = await axios.get('https://' + domain + '/api/v1/directory', {
params: {
limit: limit,
offset: page * limit,
local: true
},
timeout: getDefaultTimeoutMilliseconds()
})
assertSuccessJsonResponse(response)
const responseData = schema.parse(response.data)
if (responseData.length === 0) {
throw new Error('No more users')
}
return responseData.map(
item => {
return {
name: item.username,
displayName: replaceEmojis(item.display_name, item.emojis),
description: replaceEmojis(item.note, item.emojis),
followersCount: item.pleroma.hide_followers_count ? null : item.followers_count,
followingCount: item.pleroma.hide_follows_count ? null : item.following_count,
statusesCount: item.statuses_count,
bot: item.bot,
url: item.url,
avatar: item.avatar,
locked: item.locked,
lastStatusAt: item.last_status_at !== null ? new Date(item.last_status_at) : null,
createdAt: new Date(item.created_at),
fields: item.fields.map(field => {
return {
name: replaceEmojis(field.name, item.emojis),
value: replaceEmojis(field.value, item.emojis),
verifiedAt: null
}
}),
type: 'account',
parentFeed: null
}
}
)
} catch (error) {
throw new Error('Invalid response: ' + error)
}
}

23
application/src/Fediverse/Providers/Pleroma/retrievePeers.ts
Wyświetl plik

@ -1,23 +0,0 @@
import axios from 'axios'
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
import { z } from 'zod'
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
const schema = z.array(
z.string()
)
export const retrievePeers = async (domain:string, page:number):Promise<string[]> => {
if (page !== 0) {
throw new Error('No more peer pages')
}
try {
const response = await axios.get('https://' + domain + '/api/v1/instance/peers', {
timeout: getDefaultTimeoutMilliseconds()
})
assertSuccessJsonResponse(response)
return schema.parse(response.data)
} catch (error) {
throw new Error('Invalid response')
}
}

12
application/src/Fediverse/Providers/ProviderKeyAlreadyRegisteredError.ts 100644
Wyświetl plik

@ -0,0 +1,12 @@
export class ProviderKeyAlreadyRegisteredError extends Error {
private readonly _key:string
public constructor (key:string) {
super(`Provider with the key ${key} is already registered`)
this._key = key
}
public get key (): string {
return this._key
}
}

3
application/src/Fediverse/Providers/ProviderRegistry.ts
Wyświetl plik

@ -1,5 +1,6 @@
import { Provider } from './Provider'
import { Dictionary } from 'typescript-collections'
import { ProviderKeyAlreadyRegisteredError } from './ProviderKeyAlreadyRegisteredError'
export interface ProviderCallback {
(key: string, provider: Provider): void
@ -10,7 +11,7 @@ const providers: Dictionary<string, Provider> = new Dictionary<string, Provider>
const registerProvider = (provider: Provider): void => {
const key = provider.getKey()
if (providers.containsKey(key)) {
throw new Error(`Provider with the key ${key} is already registered`)
throw new ProviderKeyAlreadyRegisteredError(key)
}
providers.setValue(key, provider)
console.info('Added provider to registry', { key: key })

20
application/src/Fediverse/UnexpectedContentTypeError.ts 100644
Wyświetl plik

@ -0,0 +1,20 @@
import { UnexpectedResponseError } from './UnexpectedResponseError'
export class UnexpectedContentTypeError extends UnexpectedResponseError {
private readonly _expectedContentType: string
private readonly _actualContentType: string
public constructor (actualContentType: string, expectedContentType:string) {
super(`Expected content type '${expectedContentType}' but got '${actualContentType}'`)
this._expectedContentType = expectedContentType
this._actualContentType = actualContentType
}
get expectedContentType (): string {
return this._expectedContentType
}
get actualContentType (): string {
return this._actualContentType
}
}

3
application/src/Fediverse/UnexpectedResponseError.ts 100644
Wyświetl plik

@ -0,0 +1,3 @@
export class UnexpectedResponseError extends Error {
}

20
application/src/Fediverse/UnexpectedResponseStatusError.ts 100644
Wyświetl plik

@ -0,0 +1,20 @@
import { UnexpectedResponseError } from './UnexpectedResponseError'
export class UnexpectedResponseStatusError extends UnexpectedResponseError {
private readonly _expectedStatusCode: number
private readonly _actualStatusCode: number
public constructor (expectedStatusCode:number, actualStatusCode:number) {
super(`Expected response code ${expectedStatusCode} but got ${actualStatusCode}`)
this._actualStatusCode = actualStatusCode
this._expectedStatusCode = expectedStatusCode
}
get expectedStatusCode (): number {
return this._expectedStatusCode
}
get actualStatusCode (): number {
return this._actualStatusCode
}
}

16
application/src/Fediverse/assertSuccessJsonResponse.ts
Wyświetl plik

@ -1,10 +1,16 @@
import { AxiosResponse } from 'axios'
import { UnexpectedResponseStatusError } from './UnexpectedResponseStatusError'
import { UnexpectedContentTypeError } from './UnexpectedContentTypeError'
export const assertSuccessJsonResponse = (response:AxiosResponse<unknown>):void => {
if (response.status !== 200) {
throw new Error('Unexpected response ' + response.status)
export const assertSuccessJsonResponse = (response: AxiosResponse<unknown>): void => {
const expectedStatus = 200
const actualStatus = response.status
if (actualStatus !== expectedStatus) {
throw new UnexpectedResponseStatusError(expectedStatus, actualStatus)
}
if (!response.headers['content-type'].startsWith('application/json')) {
throw new Error('Unexpected content-type ' + response.headers['content-type'])
const expectedContentType = 'application/json'
const actualContentType = response.headers['content-type']
if (!actualContentType.startsWith(expectedContentType)) {
throw new UnexpectedContentTypeError(expectedContentType, actualContentType)
}
}

4
application/src/Jobs/processNextNode.ts
Wyświetl plik

@ -8,6 +8,7 @@ import { findNewNodes } from './Nodes/findNewNodes'
import { NodeProvider } from '../Fediverse/Providers/NodeProvider'
import { FeedProvider } from '../Fediverse/Providers/FeedProvider'
import { refreshFeeds } from './Feeds/refreshFeeds'
import { deleteOldFeeds } from '../Storage/Feeds/deleteOldFeeds'
export const processNextNode = async (prisma:PrismaClient, providerRegistry:ProviderRegistry):Promise<void> => {
console.info('#############################################')
@ -18,6 +19,7 @@ export const processNextNode = async (prisma:PrismaClient, providerRegistry:Prov
if (!providerRegistry.containsKey(node.softwareName)) {
console.warn('Unknown software', { domain: node.domain, software: node.softwareName })
await deleteOldFeeds(prisma, node)
await setNodeRefreshed(prisma, node)
return
}
@ -37,5 +39,7 @@ export const processNextNode = async (prisma:PrismaClient, providerRegistry:Prov
})
)
await deleteOldFeeds(prisma, node)
await setNodeRefreshed(prisma, node)
}

18
application/src/Storage/Feeds/deleteOldFeeds.ts 100644
Wyświetl plik

@ -0,0 +1,18 @@
import { Node, PrismaClient } from '@prisma/client'
export const deleteOldFeeds = async (prisma: PrismaClient, node: Node): Promise<number> => {
const result = await prisma.feed.deleteMany({
where: {
nodeId: {
equals: node.id
},
refreshedAt: {
lt: node.refreshAttemptedAt
}
}
})
console.info('Deleted old feeds', {
count: result.count, olderThen: node.refreshAttemptedAt, nodeDomain: node.domain
})
return result.count
}

5
application/src/Storage/Nodes/NoNodeFoundError.ts 100644
Wyświetl plik

@ -0,0 +1,5 @@
export class NoNodeFoundError extends Error {
public constructor () {
super('No node found')
}
}

9
application/src/Storage/Nodes/fetchNodeToProcess.ts
Wyświetl plik

@ -1,4 +1,5 @@
import { Node, PrismaClient } from '@prisma/client'
import { NoNodeFoundError } from './NoNodeFoundError'
export const fetchNodeToProcess = async (prisma: PrismaClient): Promise<Node> => {
const currentTimestamp = Date.now()
@ -56,10 +57,10 @@ export const fetchNodeToProcess = async (prisma: PrismaClient): Promise<Node> =>
]
}
})
if (node) {
console.log('Found oldest node', { domain: node.domain })
} else {
throw new Error('No node found')
if (!node) {
throw new NoNodeFoundError()
}
console.log('Found oldest node', { domain: node.domain })
return node
}