import cleanups

pull/33/head
msramalho 2022-06-03 18:30:12 +02:00
rodzic 5e495b713f
commit e2d1a5d6be
10 zmienionych plików z 26 dodań i 32 usunięć

Wyświetl plik

@ -1,8 +1,8 @@
# we need to explicitly expose the available imports here
from .base_archiver import Archiver, ArchiveResult
from .telegram_archiver import TelegramArchiver
from .telethon_archiver import TelethonArchiver, TelegramConfig
from .telethon_archiver import TelethonArchiver
from .tiktok_archiver import TiktokArchiver
from .wayback_archiver import WaybackArchiver, WaybackConfig
from .wayback_archiver import WaybackArchiver
from .youtubedl_archiver import YoutubeDLArchiver
from .twitter_archiver import TwitterArchiver

Wyświetl plik

@ -1,23 +1,16 @@
import os
import ffmpeg
import datetime
import shutil
import os, datetime, shutil, hashlib, time, requests
from dataclasses import dataclass
from abc import ABC, abstractmethod
from urllib.parse import urlparse
import hashlib
import time
import requests
import ffmpeg
from loguru import logger
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from storages import Storage
from utils import mkdir_if_not_exists
from selenium.webdriver.common.by import By
from loguru import logger
from selenium.common.exceptions import TimeoutException
@dataclass
class ArchiveResult:

Wyświetl plik

@ -1,9 +1,8 @@
import os
import requests
import os, requests, re
import html
from bs4 import BeautifulSoup
from loguru import logger
import re
import html
from .base_archiver import Archiver, ArchiveResult
from storages import Storage

Wyświetl plik

@ -1,20 +1,20 @@
import os
import re
import os, re
import html
from loguru import logger
from telethon.sync import TelegramClient
from telethon.errors import ChannelInvalidError
from storages import Storage
from .base_archiver import Archiver, ArchiveResult
from telethon.sync import TelegramClient
from telethon.errors import ChannelInvalidError
from configs import TelegramConfig
from configs import TelethonConfig
class TelethonArchiver(Archiver):
name = "telethon"
link_pattern = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+)\/(.+)")
def __init__(self, storage: Storage, driver, config: TelegramConfig):
def __init__(self, storage: Storage, driver, config: TelethonConfig):
super().__init__(storage, driver)
self.client = TelegramClient("./anon", config.api_id, config.api_hash)
@ -62,7 +62,7 @@ class TelethonArchiver(Archiver):
return False
media_posts = self._get_media_posts_in_group(chat, post)
screenshot = self.get_screenshot(url)
if len(media_posts) > 1:

Wyświetl plik

@ -5,6 +5,7 @@ from loguru import logger
from .base_archiver import Archiver, ArchiveResult
from storages import Storage
class TiktokArchiver(Archiver):
name = "tiktok"

Wyświetl plik

@ -1,11 +1,12 @@
import time, requests
from loguru import logger
from bs4 import BeautifulSoup
from storages import Storage
from .base_archiver import Archiver, ArchiveResult
from configs import WaybackConfig
from loguru import logger
class WaybackArchiver(Archiver):

Wyświetl plik

@ -1,6 +1,6 @@
import os
import datetime
import os, datetime
import yt_dlp
from loguru import logger

Wyświetl plik

@ -1,3 +1,3 @@
from .config import Config
from .wayback_config import WaybackConfig
from .telegram_config import TelegramConfig
from .telethon_config import TelethonConfig

Wyświetl plik

@ -7,7 +7,7 @@ from dataclasses import dataclass
from utils.gworksheet import GWorksheet
from .wayback_config import WaybackConfig
from .telegram_config import TelegramConfig
from .telethon_config import TelethonConfig
from storages import Storage, S3Config, S3Storage, GDStorage, GDConfig, LocalStorage
@ -131,7 +131,7 @@ class Config:
# telethon config
if "telegram" in secrets:
self.telegram_config = TelegramConfig(
self.telegram_config = TelethonConfig(
api_id=secrets["telegram"]["api_id"],
api_hash=secrets["telegram"]["api_hash"]
)

Wyświetl plik

@ -2,6 +2,6 @@
from dataclasses import dataclass
@dataclass
class TelegramConfig:
class TelethonConfig:
api_id: str
api_hash: str