kopia lustrzana https://github.com/bellingcat/auto-archiver
import cleanups
rodzic
5e495b713f
commit
e2d1a5d6be
|
@ -1,8 +1,8 @@
|
||||||
# we need to explicitly expose the available imports here
|
# we need to explicitly expose the available imports here
|
||||||
from .base_archiver import Archiver, ArchiveResult
|
from .base_archiver import Archiver, ArchiveResult
|
||||||
from .telegram_archiver import TelegramArchiver
|
from .telegram_archiver import TelegramArchiver
|
||||||
from .telethon_archiver import TelethonArchiver, TelegramConfig
|
from .telethon_archiver import TelethonArchiver
|
||||||
from .tiktok_archiver import TiktokArchiver
|
from .tiktok_archiver import TiktokArchiver
|
||||||
from .wayback_archiver import WaybackArchiver, WaybackConfig
|
from .wayback_archiver import WaybackArchiver
|
||||||
from .youtubedl_archiver import YoutubeDLArchiver
|
from .youtubedl_archiver import YoutubeDLArchiver
|
||||||
from .twitter_archiver import TwitterArchiver
|
from .twitter_archiver import TwitterArchiver
|
|
@ -1,23 +1,16 @@
|
||||||
import os
|
import os, datetime, shutil, hashlib, time, requests
|
||||||
import ffmpeg
|
|
||||||
import datetime
|
|
||||||
import shutil
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
import hashlib
|
|
||||||
import time
|
import ffmpeg
|
||||||
import requests
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from selenium.common.exceptions import TimeoutException
|
from selenium.common.exceptions import TimeoutException
|
||||||
|
from selenium.webdriver.common.by import By
|
||||||
|
|
||||||
from storages import Storage
|
from storages import Storage
|
||||||
from utils import mkdir_if_not_exists
|
from utils import mkdir_if_not_exists
|
||||||
|
|
||||||
from selenium.webdriver.common.by import By
|
|
||||||
from loguru import logger
|
|
||||||
from selenium.common.exceptions import TimeoutException
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ArchiveResult:
|
class ArchiveResult:
|
||||||
|
|
|
@ -1,9 +1,8 @@
|
||||||
import os
|
import os, requests, re
|
||||||
import requests
|
|
||||||
|
import html
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
import re
|
|
||||||
import html
|
|
||||||
|
|
||||||
from .base_archiver import Archiver, ArchiveResult
|
from .base_archiver import Archiver, ArchiveResult
|
||||||
from storages import Storage
|
from storages import Storage
|
||||||
|
|
|
@ -1,20 +1,20 @@
|
||||||
import os
|
import os, re
|
||||||
import re
|
|
||||||
import html
|
import html
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
from telethon.sync import TelegramClient
|
||||||
|
from telethon.errors import ChannelInvalidError
|
||||||
|
|
||||||
from storages import Storage
|
from storages import Storage
|
||||||
from .base_archiver import Archiver, ArchiveResult
|
from .base_archiver import Archiver, ArchiveResult
|
||||||
from telethon.sync import TelegramClient
|
from configs import TelethonConfig
|
||||||
from telethon.errors import ChannelInvalidError
|
|
||||||
from configs import TelegramConfig
|
|
||||||
|
|
||||||
|
|
||||||
class TelethonArchiver(Archiver):
|
class TelethonArchiver(Archiver):
|
||||||
name = "telethon"
|
name = "telethon"
|
||||||
link_pattern = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+)\/(.+)")
|
link_pattern = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+)\/(.+)")
|
||||||
|
|
||||||
def __init__(self, storage: Storage, driver, config: TelegramConfig):
|
def __init__(self, storage: Storage, driver, config: TelethonConfig):
|
||||||
super().__init__(storage, driver)
|
super().__init__(storage, driver)
|
||||||
self.client = TelegramClient("./anon", config.api_id, config.api_hash)
|
self.client = TelegramClient("./anon", config.api_id, config.api_hash)
|
||||||
|
|
||||||
|
|
|
@ -5,6 +5,7 @@ from loguru import logger
|
||||||
from .base_archiver import Archiver, ArchiveResult
|
from .base_archiver import Archiver, ArchiveResult
|
||||||
from storages import Storage
|
from storages import Storage
|
||||||
|
|
||||||
|
|
||||||
class TiktokArchiver(Archiver):
|
class TiktokArchiver(Archiver):
|
||||||
name = "tiktok"
|
name = "tiktok"
|
||||||
|
|
||||||
|
|
|
@ -1,11 +1,12 @@
|
||||||
import time, requests
|
import time, requests
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
from storages import Storage
|
from storages import Storage
|
||||||
from .base_archiver import Archiver, ArchiveResult
|
from .base_archiver import Archiver, ArchiveResult
|
||||||
|
|
||||||
from configs import WaybackConfig
|
from configs import WaybackConfig
|
||||||
from loguru import logger
|
|
||||||
|
|
||||||
|
|
||||||
class WaybackArchiver(Archiver):
|
class WaybackArchiver(Archiver):
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
|
|
||||||
import os
|
import os, datetime
|
||||||
import datetime
|
|
||||||
import yt_dlp
|
import yt_dlp
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,3 @@
|
||||||
from .config import Config
|
from .config import Config
|
||||||
from .wayback_config import WaybackConfig
|
from .wayback_config import WaybackConfig
|
||||||
from .telegram_config import TelegramConfig
|
from .telethon_config import TelethonConfig
|
|
@ -7,7 +7,7 @@ from dataclasses import dataclass
|
||||||
|
|
||||||
from utils.gworksheet import GWorksheet
|
from utils.gworksheet import GWorksheet
|
||||||
from .wayback_config import WaybackConfig
|
from .wayback_config import WaybackConfig
|
||||||
from .telegram_config import TelegramConfig
|
from .telethon_config import TelethonConfig
|
||||||
from storages import Storage, S3Config, S3Storage, GDStorage, GDConfig, LocalStorage
|
from storages import Storage, S3Config, S3Storage, GDStorage, GDConfig, LocalStorage
|
||||||
|
|
||||||
|
|
||||||
|
@ -131,7 +131,7 @@ class Config:
|
||||||
|
|
||||||
# telethon config
|
# telethon config
|
||||||
if "telegram" in secrets:
|
if "telegram" in secrets:
|
||||||
self.telegram_config = TelegramConfig(
|
self.telegram_config = TelethonConfig(
|
||||||
api_id=secrets["telegram"]["api_id"],
|
api_id=secrets["telegram"]["api_id"],
|
||||||
api_hash=secrets["telegram"]["api_hash"]
|
api_hash=secrets["telegram"]["api_hash"]
|
||||||
)
|
)
|
||||||
|
|
|
@ -2,6 +2,6 @@
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class TelegramConfig:
|
class TelethonConfig:
|
||||||
api_id: str
|
api_id: str
|
||||||
api_hash: str
|
api_hash: str
|
Ładowanie…
Reference in New Issue