import cleanups

pull/33/head
msramalho 2022-06-03 18:30:12 +02:00
rodzic 5e495b713f
commit e2d1a5d6be
10 zmienionych plików z 26 dodań i 32 usunięć

Wyświetl plik

@ -1,8 +1,8 @@
# we need to explicitly expose the available imports here # we need to explicitly expose the available imports here
from .base_archiver import Archiver, ArchiveResult from .base_archiver import Archiver, ArchiveResult
from .telegram_archiver import TelegramArchiver from .telegram_archiver import TelegramArchiver
from .telethon_archiver import TelethonArchiver, TelegramConfig from .telethon_archiver import TelethonArchiver
from .tiktok_archiver import TiktokArchiver from .tiktok_archiver import TiktokArchiver
from .wayback_archiver import WaybackArchiver, WaybackConfig from .wayback_archiver import WaybackArchiver
from .youtubedl_archiver import YoutubeDLArchiver from .youtubedl_archiver import YoutubeDLArchiver
from .twitter_archiver import TwitterArchiver from .twitter_archiver import TwitterArchiver

Wyświetl plik

@ -1,23 +1,16 @@
import os import os, datetime, shutil, hashlib, time, requests
import ffmpeg
import datetime
import shutil
from dataclasses import dataclass from dataclasses import dataclass
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from urllib.parse import urlparse from urllib.parse import urlparse
import hashlib
import time import ffmpeg
import requests
from loguru import logger from loguru import logger
from selenium.common.exceptions import TimeoutException from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from storages import Storage from storages import Storage
from utils import mkdir_if_not_exists from utils import mkdir_if_not_exists
from selenium.webdriver.common.by import By
from loguru import logger
from selenium.common.exceptions import TimeoutException
@dataclass @dataclass
class ArchiveResult: class ArchiveResult:

Wyświetl plik

@ -1,9 +1,8 @@
import os import os, requests, re
import requests
import html
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from loguru import logger from loguru import logger
import re
import html
from .base_archiver import Archiver, ArchiveResult from .base_archiver import Archiver, ArchiveResult
from storages import Storage from storages import Storage

Wyświetl plik

@ -1,20 +1,20 @@
import os import os, re
import re
import html import html
from loguru import logger from loguru import logger
from telethon.sync import TelegramClient
from telethon.errors import ChannelInvalidError
from storages import Storage from storages import Storage
from .base_archiver import Archiver, ArchiveResult from .base_archiver import Archiver, ArchiveResult
from telethon.sync import TelegramClient from configs import TelethonConfig
from telethon.errors import ChannelInvalidError
from configs import TelegramConfig
class TelethonArchiver(Archiver): class TelethonArchiver(Archiver):
name = "telethon" name = "telethon"
link_pattern = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+)\/(.+)") link_pattern = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+)\/(.+)")
def __init__(self, storage: Storage, driver, config: TelegramConfig): def __init__(self, storage: Storage, driver, config: TelethonConfig):
super().__init__(storage, driver) super().__init__(storage, driver)
self.client = TelegramClient("./anon", config.api_id, config.api_hash) self.client = TelegramClient("./anon", config.api_id, config.api_hash)

Wyświetl plik

@ -5,6 +5,7 @@ from loguru import logger
from .base_archiver import Archiver, ArchiveResult from .base_archiver import Archiver, ArchiveResult
from storages import Storage from storages import Storage
class TiktokArchiver(Archiver): class TiktokArchiver(Archiver):
name = "tiktok" name = "tiktok"

Wyświetl plik

@ -1,11 +1,12 @@
import time, requests import time, requests
from loguru import logger
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from storages import Storage from storages import Storage
from .base_archiver import Archiver, ArchiveResult from .base_archiver import Archiver, ArchiveResult
from configs import WaybackConfig from configs import WaybackConfig
from loguru import logger
class WaybackArchiver(Archiver): class WaybackArchiver(Archiver):

Wyświetl plik

@ -1,6 +1,6 @@
import os import os, datetime
import datetime
import yt_dlp import yt_dlp
from loguru import logger from loguru import logger

Wyświetl plik

@ -1,3 +1,3 @@
from .config import Config from .config import Config
from .wayback_config import WaybackConfig from .wayback_config import WaybackConfig
from .telegram_config import TelegramConfig from .telethon_config import TelethonConfig

Wyświetl plik

@ -7,7 +7,7 @@ from dataclasses import dataclass
from utils.gworksheet import GWorksheet from utils.gworksheet import GWorksheet
from .wayback_config import WaybackConfig from .wayback_config import WaybackConfig
from .telegram_config import TelegramConfig from .telethon_config import TelethonConfig
from storages import Storage, S3Config, S3Storage, GDStorage, GDConfig, LocalStorage from storages import Storage, S3Config, S3Storage, GDStorage, GDConfig, LocalStorage
@ -131,7 +131,7 @@ class Config:
# telethon config # telethon config
if "telegram" in secrets: if "telegram" in secrets:
self.telegram_config = TelegramConfig( self.telegram_config = TelethonConfig(
api_id=secrets["telegram"]["api_id"], api_id=secrets["telegram"]["api_id"],
api_hash=secrets["telegram"]["api_hash"] api_hash=secrets["telegram"]["api_hash"]
) )

Wyświetl plik

@ -2,6 +2,6 @@
from dataclasses import dataclass from dataclasses import dataclass
@dataclass @dataclass
class TelegramConfig: class TelethonConfig:
api_id: str api_id: str
api_hash: str api_hash: str