diff --git a/src/auto_archiver/modules/generic_extractor/twitter.py b/src/auto_archiver/modules/generic_extractor/twitter.py index e27a0c1..189a7e6 100644 --- a/src/auto_archiver/modules/generic_extractor/twitter.py +++ b/src/auto_archiver/modules/generic_extractor/twitter.py @@ -1,6 +1,5 @@ import re import mimetypes -import json from loguru import logger from slugify import slugify @@ -32,6 +31,9 @@ class Twitter(GenericDropin): twid = ie_instance._match_valid_url(url).group("id") return ie_instance._extract_status(twid=twid) + def keys_to_clean(self, video_data, info_extractor): + return ["user", "created_at", "entities", "favorited", "translator_type"] + def create_metadata(self, tweet: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata: result = Metadata() try: @@ -42,9 +44,11 @@ class Twitter(GenericDropin): logger.warning(f"Unable to parse tweet: {str(ex)}\nRetreived tweet data: {tweet}") return False - result.set_title(tweet.get("full_text", "")).set_content(json.dumps(tweet, ensure_ascii=False)).set_timestamp( - timestamp - ) + full_text = tweet.pop("full_text", "") + author = tweet["user"].get("name", "") + result.set("author", author).set_url(url) + + result.set_title(f"{author} - {full_text}").set_content(full_text).set_timestamp(timestamp) if not tweet.get("entities", {}).get("media"): logger.debug("No media found, archiving tweet text only") result.status = "twitter-ytdl" diff --git a/tests/extractors/test_generic_extractor.py b/tests/extractors/test_generic_extractor.py index 2089007..616183b 100644 --- a/tests/extractors/test_generic_extractor.py +++ b/tests/extractors/test_generic_extractor.py @@ -206,10 +206,11 @@ class TestGenericExtractor(TestExtractorBase): self.assertValidResponseMetadata( post, - "Onion rings are just vegetable donuts.", + "Cookie Monster - Onion rings are just vegetable donuts.", datetime.datetime(2023, 1, 24, 16, 25, 51, tzinfo=datetime.timezone.utc), "yt-dlp_Twitter: success", ) + assert post.get("content") == "Onion rings are just vegetable donuts." @pytest.mark.download def test_twitter_download_video(self, make_item):