diff --git a/.gitignore b/.gitignore index 9d83858..a59087a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ tmp/ .DS_Store expmt/ service_account.json +drive_service_account.json __pycache__/ ._* anu.html diff --git a/Pipfile b/Pipfile index 0f20e24..48f1429 100644 --- a/Pipfile +++ b/Pipfile @@ -17,8 +17,13 @@ selenium = "*" snscrape = "*" yt-dlp = "*" telethon = "*" +google-api-python-client = "*" +google-auth-httplib2 = "*" +google-auth-oauthlib = "*" +oauth2client = "*" [dev-packages] +autopep8 = "*" [requires] python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock index d269c6c..bd5142a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "e27ea0a6fdf6e588c14fbb90af45f784b9e55a9b986a3b50770490648ba96720" + "sha256": "204bac9d742ba5f55c2a4d3501b11d53a5112e58f02232684e7542185136b1c4" }, "pipfile-spec": 6, "requires": { @@ -50,19 +50,19 @@ }, "boto3": { "hashes": [ - "sha256:895fb88c69be78f82cfee58a79c97a3ad8d4a2a1209041a411d7d6b9fc5393e4", - "sha256:bcb541175a7d190dd919a0af0e807ee6e9d26f135551e741b10d94343f2d7588" + "sha256:1d0c9cf30c5c93ce4628b22f7dff21f80efd104706967be2afb47ea93dcbceb8", + "sha256:fb9a66722a15b0a8edb4f29d63f013877c4c528eff1225bb31dab723c3e5a9ed" ], "index": "pypi", - "version": "==1.21.42" + "version": "==1.22.2" }, "botocore": { "hashes": [ - "sha256:14aee41c8bf59d2dd2d89e8751fa37d3c95dcb92707d1966aa02697e914c1417", - "sha256:a2baa9484bbaee96ef312c049b8e360badcab58329e487b57567644a571b5f4a" + "sha256:78363cd023c6123a65d90080e14a9fb625a9639d970ab03471c6fe95258ab8c0", + "sha256:dff4aaf03c2b8ee4a861405fd35c83b585a661a541d520f51e863a31bb18e0f8" ], - "markers": "python_version >= '3.6'", - "version": "==1.24.42" + "markers": "python_full_version >= '3.6.0'", + "version": "==1.25.2" }, "brotli": { "hashes": [ @@ -234,36 +234,38 @@ }, "cryptography": { "hashes": [ - "sha256:0a3bf09bb0b7a2c93ce7b98cb107e9170a90c51a0162a20af1c61c765b90e60b", - "sha256:1f64a62b3b75e4005df19d3b5235abd43fa6358d5516cfc43d87aeba8d08dd51", - "sha256:32db5cc49c73f39aac27574522cecd0a4bb7384e71198bc65a0d23f901e89bb7", - "sha256:4881d09298cd0b669bb15b9cfe6166f16fc1277b4ed0d04a22f3d6430cb30f1d", - "sha256:4e2dddd38a5ba733be6a025a1475a9f45e4e41139d1321f412c6b360b19070b6", - "sha256:53e0285b49fd0ab6e604f4c5d9c5ddd98de77018542e88366923f152dbeb3c29", - "sha256:70f8f4f7bb2ac9f340655cbac89d68c527af5bb4387522a8413e841e3e6628c9", - "sha256:7b2d54e787a884ffc6e187262823b6feb06c338084bbe80d45166a1cb1c6c5bf", - "sha256:7be666cc4599b415f320839e36367b273db8501127b38316f3b9f22f17a0b815", - "sha256:8241cac0aae90b82d6b5c443b853723bcc66963970c67e56e71a2609dc4b5eaf", - "sha256:82740818f2f240a5da8dfb8943b360e4f24022b093207160c77cadade47d7c85", - "sha256:8897b7b7ec077c819187a123174b645eb680c13df68354ed99f9b40a50898f77", - "sha256:c2c5250ff0d36fd58550252f54915776940e4e866f38f3a7866d92b32a654b86", - "sha256:ca9f686517ec2c4a4ce930207f75c00bf03d94e5063cbc00a1dc42531511b7eb", - "sha256:d2b3d199647468d410994dbeb8cec5816fb74feb9368aedf300af709ef507e3e", - "sha256:da73d095f8590ad437cd5e9faf6628a218aa7c387e1fdf67b888b47ba56a17f0", - "sha256:e167b6b710c7f7bc54e67ef593f8731e1f45aa35f8a8a7b72d6e42ec76afd4b3", - "sha256:ea634401ca02367c1567f012317502ef3437522e2fc44a3ea1844de028fa4b84", - "sha256:ec6597aa85ce03f3e507566b8bcdf9da2227ec86c4266bd5e6ab4d9e0cc8dab2", - "sha256:f64b232348ee82f13aac22856515ce0195837f6968aeaa94a3d0353ea2ec06a6" + "sha256:06bfafa6e53ccbfb7a94be4687b211a025ce0625e3f3c60bb15cd048a18f3ed8", + "sha256:0db5cf21bd7d092baacb576482b0245102cea2d3cf09f09271ce9f69624ecb6f", + "sha256:125702572be12bcd318e3a14e9e70acd4be69a43664a75f0397e8650fe3c6cc3", + "sha256:1858eff6246bb8bbc080eee78f3dd1528739e3f416cba5f9914e8631b8df9871", + "sha256:315af6268de72bcfa0bb3401350ce7d921f216e6b60de12a363dad128d9d459f", + "sha256:451aaff8b8adf2dd0597cbb1fdcfc8a7d580f33f843b7cce75307a7f20112dd8", + "sha256:58021d6e9b1d88b1105269d0da5e60e778b37dfc0e824efc71343dd003726831", + "sha256:618391152147a1221c87b1b0b7f792cafcfd4b5a685c5c72eeea2ddd29aeceff", + "sha256:6d4daf890e674d191757d8d7d60dc3a29c58c72c7a76a05f1c0a326013f47e8b", + "sha256:74b55f67f4cf026cb84da7a1b04fc2a1d260193d4ad0ea5e9897c8b74c1e76ac", + "sha256:7ceae26f876aabe193b13a0c36d1bb8e3e7e608d17351861b437bd882f617e9f", + "sha256:930b829e8a2abaf43a19f38277ae3c5e1ffcf547b936a927d2587769ae52c296", + "sha256:a18ff4bfa9d64914a84d7b06c46eb86e0cc03113470b3c111255aceb6dcaf81d", + "sha256:ae1cd29fbe6b716855454e44f4bf743465152e15d2d317303fe3b58ee9e5af7a", + "sha256:b1ee5c82cf03b30f6ae4e32d2bcb1e167ef74d6071cbb77c2af30f101d0b360b", + "sha256:bf585476fcbcd37bed08072e8e2db3954ce1bfc68087a2dc9c19cfe0b90979ca", + "sha256:c4a58eeafbd7409054be41a377e726a7904a17c26f45abf18125d21b1215b08b", + "sha256:cce90609e01e1b192fae9e13665058ab46b2ea53a3c05a3ea74a3eb8c3af8857", + "sha256:d610d0ee14dd9109006215c7c0de15eee91230b70a9bce2263461cf7c3720b83", + "sha256:e69a0e36e62279120e648e787b76d79b41e0f9e86c1c636a4f38d415595c722e", + "sha256:f095988548ec5095e3750cdb30e6962273d239b1998ba1aac66c0d5bee7111c1", + "sha256:faf0f5456c059c7b1c29441bdd5e988f0ba75bdc3eea776520d8dcb1e30e1b5c" ], - "version": "==36.0.2" + "version": "==37.0.1" }, "faker": { "hashes": [ - "sha256:188961065fb5c78ea639f42176f55100f72c90c3a3179ac6c955c4bd712b0511", - "sha256:7758ece2593ce603db117db3d27393c31f4af03f783e176f3f0e14839a4f3426" + "sha256:0d5425894e098410b64aaade38a81074fa30163076251118523adf5bb44f8346", + "sha256:7ab2f741ef1c006ed7274a6ed75695ca8b610f78861566b599ce83c4953bf687" ], - "markers": "python_version >= '3.6'", - "version": "==13.3.4" + "markers": "python_full_version >= '3.6.0'", + "version": "==13.6.0" }, "ffmpeg-python": { "hashes": [ @@ -296,22 +298,54 @@ "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.18.2" }, + "google-api-core": { + "hashes": [ + "sha256:65480309a7437f739e4476da037af02a3ec8263f1d1f89f72bbdc8f54fe402d2", + "sha256:8fcbe52dc129fd83dca4e638a76f22b3a11579c493e643134e50e9870b233302" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==2.7.2" + }, + "google-api-python-client": { + "hashes": [ + "sha256:1a7a954fdb778c77ad2a7b28c8e5f4b8761728aca89f5a017e05abd92814b6e9", + "sha256:ba41ce1da337d39d76da1c5affea6e0b95bb6da74aa2adfe5a40618dd67a01fe" + ], + "index": "pypi", + "version": "==2.46.0" + }, "google-auth": { "hashes": [ - "sha256:04e224f241c0566477bb35a8a93be8c635210de743bde454d49393cfb605266d", - "sha256:9a88ee548f6fd49467e2e443dfbfe10344e5a270629a137a3a0b3437ec6b02a6" + "sha256:1ba4938e032b73deb51e59c4656a00e0939cf0b1112575099f136babb4563312", + "sha256:349ac49b18b01019453cc99c11c92ed772739778c92f184002b7ab3a5b7ac77d" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==2.6.5" + "version": "==2.6.6" + }, + "google-auth-httplib2": { + "hashes": [ + "sha256:31e49c36c6b5643b57e82617cb3e021e3e1d2df9da63af67252c02fa9c1f4a10", + "sha256:a07c39fd632becacd3f07718dfd6021bf396978f03ad3ce4321d060015cc30ac" + ], + "index": "pypi", + "version": "==0.1.0" }, "google-auth-oauthlib": { "hashes": [ "sha256:24f67735513c4c7134dbde2f1dee5a1deb6acc8dfcb577d7bff30d213a28e7b0", "sha256:30596b824fc6808fdaca2f048e4998cc40fb4b3599eaea66d28dc7085b36c5b8" ], - "markers": "python_version >= '3.6'", + "index": "pypi", "version": "==0.5.1" }, + "googleapis-common-protos": { + "hashes": [ + "sha256:4007500795bcfc269d279f0f7d253ae18d6dc1ff5d5a73613ffe452038b1ec5f", + "sha256:60220c89b8bd5272159bed4929ecdc1243ae1f73437883a499a44a1cbc084086" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==1.56.0" + }, "gspread": { "hashes": [ "sha256:319766d90db05056293f7ee0ad2b35503a1a40683a75897a2922398cd2016283", @@ -325,9 +359,17 @@ "sha256:70813c1135087a248a4d38cc0e1a0181ffab2188141a93eaf567940c3957ff06", "sha256:8ddd78563b633ca55346c8cd41ec0af27d3c79931828beffb46ce70a379e7442" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==0.13.0" }, + "httplib2": { + "hashes": [ + "sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585", + "sha256:8b6a905cb1c79eefd03f8669fd993c36dc341f7c558f056cb5a33b5c2f458543" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.20.4" + }, "idna": { "hashes": [ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", @@ -497,12 +539,20 @@ "markers": "python_version >= '3.5' and python_version < '4'", "version": "==1.45.1" }, + "oauth2client": { + "hashes": [ + "sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac", + "sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6" + ], + "index": "pypi", + "version": "==4.1.3" + }, "oauthlib": { "hashes": [ "sha256:23a8208d75b902797ea29fd31fa80a15ed9dc2c6c16fe73f5d346f83f6fa27a2", "sha256:6db33440354787f9b7f3a6dbd4febf5d0f93758354060e802f6c06cb493022fe" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==3.2.0" }, "outcome": { @@ -510,9 +560,39 @@ "sha256:c7dd9375cfd3c12db9801d080a3b63d4b0a261aa996c4c13152380587288d958", "sha256:e862f01d4e626e63e8f92c38d1f8d5546d3f9cce989263c521b2e7990d186967" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==1.1.0" }, + "protobuf": { + "hashes": [ + "sha256:06059eb6953ff01e56a25cd02cca1a9649a75a7e65397b5b9b4e929ed71d10cf", + "sha256:097c5d8a9808302fb0da7e20edf0b8d4703274d140fd25c5edabddcde43e081f", + "sha256:284f86a6207c897542d7e956eb243a36bb8f9564c1742b253462386e96c6b78f", + "sha256:32ca378605b41fd180dfe4e14d3226386d8d1b002ab31c969c366549e66a2bb7", + "sha256:3cc797c9d15d7689ed507b165cd05913acb992d78b379f6014e013f9ecb20996", + "sha256:62f1b5c4cd6c5402b4e2d63804ba49a327e0c386c99b1675c8a0fefda23b2067", + "sha256:69ccfdf3657ba59569c64295b7d51325f91af586f8d5793b734260dfe2e94e2c", + "sha256:6f50601512a3d23625d8a85b1638d914a0970f17920ff39cec63aaef80a93fb7", + "sha256:7403941f6d0992d40161aa8bb23e12575637008a5a02283a930addc0508982f9", + "sha256:755f3aee41354ae395e104d62119cb223339a8f3276a0cd009ffabfcdd46bb0c", + "sha256:77053d28427a29987ca9caf7b72ccafee011257561259faba8dd308fda9a8739", + "sha256:7e371f10abe57cee5021797126c93479f59fccc9693dafd6bd5633ab67808a91", + "sha256:9016d01c91e8e625141d24ec1b20fed584703e527d28512aa8c8707f105a683c", + "sha256:9be73ad47579abc26c12024239d3540e6b765182a91dbc88e23658ab71767153", + "sha256:adc31566d027f45efe3f44eeb5b1f329da43891634d61c75a5944e9be6dd42c9", + "sha256:adfc6cf69c7f8c50fd24c793964eef18f0ac321315439d94945820612849c388", + "sha256:af0ebadc74e281a517141daad9d0f2c5d93ab78e9d455113719a45a49da9db4e", + "sha256:cb29edb9eab15742d791e1025dd7b6a8f6fcb53802ad2f6e3adcb102051063ab", + "sha256:cd68be2559e2a3b84f517fb029ee611546f7812b1fdd0aa2ecc9bc6ec0e4fdde", + "sha256:cdee09140e1cd184ba9324ec1df410e7147242b94b5f8b0c64fc89e38a8ba531", + "sha256:db977c4ca738dd9ce508557d4fce0f5aebd105e158c725beec86feb1f6bc20d8", + "sha256:dd5789b2948ca702c17027c84c2accb552fc30f4622a98ab5c51fcfe8c50d3e7", + "sha256:e250a42f15bf9d5b09fe1b293bdba2801cd520a9f5ea2d7fb7536d4441811d20", + "sha256:ff8d8fa42675249bb456f5db06c00de6c2f4c27a065955917b28c4f15978b9c3" + ], + "markers": "python_version >= '3.7'", + "version": "==3.20.1" + }, "pyaes": { "hashes": [ "sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f" @@ -662,7 +742,7 @@ "sha256:5c6bd9dc7a543b7fe4304a631f8a8a3b674e2bbfc49c2ae96200cdbe55df6b17", "sha256:95c5d300c4e879ee69708c428ba566c59478fd653cc3a22243eeb8ed846950bb" ], - "markers": "python_version >= '3.6' and python_version < '4'", + "markers": "python_version < '4' and python_full_version >= '3.6.0'", "version": "==4.8" }, "s3transfer": { @@ -670,7 +750,7 @@ "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971", "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==0.5.2" }, "selenium": { @@ -716,7 +796,7 @@ "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759", "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.3.2.post1" }, "telethon": { @@ -747,7 +827,19 @@ "markers": "python_version >= '3.5'", "version": "==0.9.2" }, + "uritemplate": { + "hashes": [ + "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0", + "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==4.1.1" + }, "urllib3": { + "extras": [ + "secure", + "socks" + ], "hashes": [ "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" @@ -842,5 +934,30 @@ "version": "==3.8.0" } }, - "develop": {} + "develop": { + "autopep8": { + "hashes": [ + "sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979", + "sha256:ed77137193bbac52d029a52c59bec1b0629b5a186c495f1eb21b126ac466083f" + ], + "index": "pypi", + "version": "==1.6.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", + "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.8.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + } + } } diff --git a/README.md b/README.md index fe92d13..1b0b11b 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,8 @@ not tested as the API is getting all so far # TikTok -not tested yet +Always getting invalid URL using the library [https://github.com/msramalho/tiktok-downloader](https://github.com/msramalho/tiktok-downloader) which is a fork of [https://github.com/krypton-byte/tiktok-downloader](https://github.com/krypton-byte/tiktok-downloader) and has a nice test app [https://tkdown.herokuapp.com/](https://tkdown.herokuapp.com/) + # Twitter Video - YoutubeDL diff --git a/abc.txt b/abc.txt new file mode 100644 index 0000000..fe0c852 --- /dev/null +++ b/abc.txt @@ -0,0 +1 @@ +sdf \ No newline at end of file diff --git a/archivers/telegram_archiver.py b/archivers/telegram_archiver.py index fdb6417..773fa11 100644 --- a/archivers/telegram_archiver.py +++ b/archivers/telegram_archiver.py @@ -11,7 +11,7 @@ from .base_archiver import Archiver, ArchiveResult class TelegramArchiver(Archiver): name = "telegram" - def download(self, url, check_if_exists=False, filenumber=""): + def download(self, url, check_if_exists=False, filenumber=None): # detect URLs that we definitely cannot handle if 't.me' != self.get_netloc(url): return False @@ -52,6 +52,10 @@ class TelegramArchiver(Archiver): video_id = video_url.split('/')[-1].split('?')[0] key = self.get_key(video_id) + # DM feature flag (not tested as telethon gets all requests) + if filenumber is not None: + key = filenumber + "/" + key + filename = 'tmp/' + key cdn_url = self.storage.get_cdn_url(key) @@ -81,7 +85,7 @@ class TelegramArchiver(Archiver): # process thumbnails key_thumb, thumb_index = self.get_thumbnails( - filename, key, duration=duration) + filename, key, duration=duration, filenumber=filenumber) os.remove(filename) return ArchiveResult(status=status, cdn_url=cdn_url, thumbnail=key_thumb, thumbnail_index=thumb_index, diff --git a/archivers/telethon_archiver.py b/archivers/telethon_archiver.py index e8abd2d..6e83051 100644 --- a/archivers/telethon_archiver.py +++ b/archivers/telethon_archiver.py @@ -48,7 +48,7 @@ class TelethonArchiver(Archiver): return False status = "success" - screenshot = self.get_screenshot(url) + screenshot = self.get_screenshot(url, filenumber) # app will ask (stall for user input!) for phone number and auth code if anon.session not found with self.client.start(): @@ -67,6 +67,11 @@ class TelethonArchiver(Archiver): if len(media_posts) > 1: key = self.get_html_key(url) + + # DM feature flag + if filenumber is not None: + key = filenumber + "/" + key + cdn_url = self.storage.get_cdn_url(key) if check_if_exists and self.storage.exists(key): @@ -80,26 +85,33 @@ class TelethonArchiver(Archiver): if len(mp.message) > len(message): message = mp.message filename = self.client.download_media(mp.media, f'tmp/{chat}_{group_id}/{mp.id}') key = filename.split('tmp/')[1] + # DM feature flag + if filenumber is not None: + key = filenumber + "/" + key self.storage.upload(filename, key) hash = self.get_hash(filename) cdn_url = self.storage.get_cdn_url(key) uploaded_media.append({'cdn_url': cdn_url, 'key': key, 'hash': hash}) os.remove(filename) - page_cdn, page_hash, _ = self.generate_media_page_html(url, uploaded_media, html.escape(str(post))) + #DM + page_cdn, page_hash, _ = self.generate_media_page_html(url, uploaded_media, html.escape(str(post)), filenumber=filenumber) return ArchiveResult(status=status, cdn_url=page_cdn, title=post.message, timestamp=post.date, hash=page_hash, screenshot=screenshot) elif len(media_posts) == 1: key = self.get_key(f'{chat}_{post_id}') filename = self.client.download_media(post.media, f'tmp/{key}') key = filename.split('tmp/')[1].replace(" ", "") + # DM feature flag + if filenumber is not None: + key = filenumber + "/" + key self.storage.upload(filename, key) hash = self.get_hash(filename) cdn_url = self.storage.get_cdn_url(key) - key_thumb, thumb_index = self.get_thumbnails(filename, key) + key_thumb, thumb_index = self.get_thumbnails(filename, key, filenumber=filenumber) os.remove(filename) return ArchiveResult(status=status, cdn_url=cdn_url, title=post.message, thumbnail=key_thumb, thumbnail_index=thumb_index, timestamp=post.date, hash=hash, screenshot=screenshot) - page_cdn, page_hash, _ = self.generate_media_page_html(url, [], html.escape(str(post))) + page_cdn, page_hash, _ = self.generate_media_page_html(url, [], html.escape(str(post)), filenumber=filenumber) return ArchiveResult(status=status, cdn_url=page_cdn, title=post.message, timestamp=post.date, hash=page_hash, screenshot=screenshot) diff --git a/archivers/tiktok_archiver.py b/archivers/tiktok_archiver.py index b9c1470..aa5c9ec 100644 --- a/archivers/tiktok_archiver.py +++ b/archivers/tiktok_archiver.py @@ -37,7 +37,7 @@ class TiktokArchiver(Archiver): self.storage.upload(filename, key) try: - key_thumb, thumb_index = self.get_thumbnails(filename, key, duration=info.duration) + key_thumb, thumb_index = self.get_thumbnails(filename, key, duration=info.duration, filenumber=filenumber) except Exception as e: logger.error(e) key_thumb = '' @@ -54,8 +54,9 @@ class TiktokArchiver(Archiver): thumbnail_index=thumb_index, duration=info.duration, title=info.caption, timestamp=info.create.isoformat(), hash=hash, screenshot=screenshot) - except tiktok_downloader.Except.InvalidUrl: + except tiktok_downloader.Except.InvalidUrl as e: status = 'Invalid URL' + logger.warning(f'Invalid URL on {url} {e}\n{traceback.format_exc()}') return ArchiveResult(status=status) except: diff --git a/auto_archive.py b/auto_archive.py index 7a128cc..6181fc3 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -154,8 +154,13 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES, usefilenumbe logger.debug(f'Trying {archiver} on row {row}') try: - # DM . filenumber="" if not to be used - result = archiver.download(url, check_if_exists=True, filenumber=filenumber) + # DM + if usefilenumber: + # using filenumber to store in folders so can't check for existance of that url + result = archiver.download(url, check_if_exists=False, filenumber=filenumber) + else: + result = archiver.download(url, check_if_exists=True, filenumber=filenumber) + except Exception as e: result = False logger.error(f'Got unexpected error in row {row} with archiver {archiver} for url {url}: {e}\n{traceback.format_exc()}') diff --git a/dm_drive.py b/dm_drive.py new file mode 100644 index 0000000..a7e3ffa --- /dev/null +++ b/dm_drive.py @@ -0,0 +1,59 @@ +from __future__ import print_function + +import os.path + +from google.auth.transport.requests import Request +from google.oauth2.credentials import Credentials +from google_auth_oauthlib.flow import InstalledAppFlow +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + +# If modifying these scopes, delete the file token.json. +SCOPES = ['https://www.googleapis.com/auth/drive.metadata.readonly'] + + +def main(): + + + """Shows basic usage of the Drive v3 API. + Prints the names and ids of the first 10 files the user has access to. + """ + creds = None + # The file token.json stores the user's access and refresh tokens, and is + # created automatically when the authorization flow completes for the first + # time. + if os.path.exists('token.json'): + creds = Credentials.from_authorized_user_file('token.json', SCOPES) + # If there are no (valid) credentials available, let the user log in. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file( + 'credentials.json', SCOPES) + creds = flow.run_local_server(port=0) + # Save the credentials for the next run + with open('token.json', 'w') as token: + token.write(creds.to_json()) + + try: + service = build('drive', 'v3', credentials=creds) + + # Call the Drive v3 API + results = service.files().list( + pageSize=10, fields="nextPageToken, files(id, name)").execute() + items = results.get('files', []) + + if not items: + print('No files found.') + return + print('Files:') + for item in items: + print(u'{0} ({1})'.format(item['name'], item['id'])) + except HttpError as error: + # TODO(developer) - Handle errors from drive API. + print(f'An error occurred: {error}') + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/dm_drive2.py b/dm_drive2.py new file mode 100644 index 0000000..1a1923c --- /dev/null +++ b/dm_drive2.py @@ -0,0 +1,34 @@ +from google.oauth2 import service_account + +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + +def main(): + # SCOPES = ['https://www.googleapis.com/auth/drive.metadata.readonly'] + SCOPES = ['https://www.googleapis.com/auth/drive'] + + creds = service_account.Credentials.from_service_account_file('service_account.json', scopes=SCOPES) + + try: + service = build('drive', 'v3', credentials=creds) + + # Call the Drive v3 API + # results = service.files().list(pageSize=20, fields="nextPageToken, files(id, name)").execute() + results = service.files().list().execute() + items = results.get('files', []) + + if not items: + print('No files found.') + return + + print('Files:') + for item in items: + print(u'{0} ({1})'.format(item['name'], item['id'])) + + except HttpError as error: + # TODO(developer) - Handle errors from drive API. + print(f'An error occurred: {error}') + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/dm_drive3_upload.py b/dm_drive3_upload.py new file mode 100644 index 0000000..8662dbc --- /dev/null +++ b/dm_drive3_upload.py @@ -0,0 +1,82 @@ + +from __future__ import print_function + +import google.auth +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError +from googleapiclient.http import MediaFileUpload +from google.oauth2 import service_account + + +def upload_appdata(): + # creds, _ = google.auth.default() + SCOPES = ['https://www.googleapis.com/auth/drive'] + + creds = service_account.Credentials.from_service_account_file('service_account.json', scopes=SCOPES) + + try: + # call drive api client + service = build('drive', 'v3', credentials=creds) + + # create a file in a new folder + # file_metadata = { + # 'name': 'Invoices', + # 'mimeType': 'application/vnd.google-apps.folder' + # } + # file = service.files().create(body=file_metadata, + # fields='id').execute() + # print('Folder ID: %s' % file.get('id')) + + # upload an image + file_metadata = {'name': 'photo.jpg'} + media = MediaFileUpload('files/photo.jpg', + mimetype='image/jpeg') + file = service.files().create(body=file_metadata, + media_body=media, + fields='id').execute() + id = file.get('id') + print(f'id: {id}') + + # list files and folders + results = service.files().list().execute() + items = results.get('files', []) + + for item in items: + print(u'{0} ({1})'.format(item['name'], item['id'])) + + # upload an image to a folder + folder_id ='1ljwzoAdKdJMJzRW9gPHDC8fkRykVH83X' + file_metadata = { + 'name': 'photo.jpg', + 'parents': [folder_id] + } + media = MediaFileUpload('files/photo.jpg', + mimetype='image/jpeg', + resumable=True) + file = service.files().create(body=file_metadata, + media_body=media, + fields='id').execute() + + # print 'File ID: %s' % file.get('id') + + # pylint: disable=maybe-no-member + # file_metadata = { + # 'name': 'abc.txt', + # 'parents': ['appDataFolder'] + # } + # media = MediaFileUpload('abc.txt', + # mimetype='text/txt', + # resumable=True) + # file = service.files().create(body=file_metadata, media_body=media, + # fields='id').execute() + # print(F'File ID: {file.get("id")}') + + except HttpError as error: + print(F'An error occurred: {error}') + file = None + + return file.get('id') + + +if __name__ == '__main__': + upload_appdata() \ No newline at end of file diff --git a/files/photo.jpg b/files/photo.jpg new file mode 100644 index 0000000..62d5d62 Binary files /dev/null and b/files/photo.jpg differ