mirror
/
auto-archiver
kopia lustrzana https://github.com/bellingcat/auto-archiver
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

auto

pull/32/head
Dave Mateer 2022-04-28 19:28:34 +01:00
rodzic 22c1fb09fd
commit 6bca0d979c
13 zmienionych plików z 377 dodań i 55 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

1
.gitignore vendored
Wyświetl plik

@ -3,6 +3,7 @@ tmp/
.DS_Store
expmt/
service_account.json
drive_service_account.json
__pycache__/
._*
anu.html

5
Pipfile
Wyświetl plik

@ -17,8 +17,13 @@ selenium = "*"
snscrape = "*"
yt-dlp = "*"
telethon = "*"
google-api-python-client = "*"
google-auth-httplib2 = "*"
google-auth-oauthlib = "*"
oauth2client = "*"
[dev-packages]
autopep8 = "*"
[requires]
python_version = "3.9"

205
Pipfile.lock wygenerowano
Wyświetl plik

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "e27ea0a6fdf6e588c14fbb90af45f784b9e55a9b986a3b50770490648ba96720"
"sha256": "204bac9d742ba5f55c2a4d3501b11d53a5112e58f02232684e7542185136b1c4"
},
"pipfile-spec": 6,
"requires": {
@ -50,19 +50,19 @@
},
"boto3": {
"hashes": [
"sha256:895fb88c69be78f82cfee58a79c97a3ad8d4a2a1209041a411d7d6b9fc5393e4",
"sha256:bcb541175a7d190dd919a0af0e807ee6e9d26f135551e741b10d94343f2d7588"
"sha256:1d0c9cf30c5c93ce4628b22f7dff21f80efd104706967be2afb47ea93dcbceb8",
"sha256:fb9a66722a15b0a8edb4f29d63f013877c4c528eff1225bb31dab723c3e5a9ed"
],
"index": "pypi",
"version": "==1.21.42"
"version": "==1.22.2"
},
"botocore": {
"hashes": [
"sha256:14aee41c8bf59d2dd2d89e8751fa37d3c95dcb92707d1966aa02697e914c1417",
"sha256:a2baa9484bbaee96ef312c049b8e360badcab58329e487b57567644a571b5f4a"
"sha256:78363cd023c6123a65d90080e14a9fb625a9639d970ab03471c6fe95258ab8c0",
"sha256:dff4aaf03c2b8ee4a861405fd35c83b585a661a541d520f51e863a31bb18e0f8"
],
"markers": "python_version >= '3.6'",
"version": "==1.24.42"
"markers": "python_full_version >= '3.6.0'",
"version": "==1.25.2"
},
"brotli": {
"hashes": [
@ -234,36 +234,38 @@
},
"cryptography": {
"hashes": [
"sha256:0a3bf09bb0b7a2c93ce7b98cb107e9170a90c51a0162a20af1c61c765b90e60b",
"sha256:1f64a62b3b75e4005df19d3b5235abd43fa6358d5516cfc43d87aeba8d08dd51",
"sha256:32db5cc49c73f39aac27574522cecd0a4bb7384e71198bc65a0d23f901e89bb7",
"sha256:4881d09298cd0b669bb15b9cfe6166f16fc1277b4ed0d04a22f3d6430cb30f1d",
"sha256:4e2dddd38a5ba733be6a025a1475a9f45e4e41139d1321f412c6b360b19070b6",
"sha256:53e0285b49fd0ab6e604f4c5d9c5ddd98de77018542e88366923f152dbeb3c29",
"sha256:70f8f4f7bb2ac9f340655cbac89d68c527af5bb4387522a8413e841e3e6628c9",
"sha256:7b2d54e787a884ffc6e187262823b6feb06c338084bbe80d45166a1cb1c6c5bf",
"sha256:7be666cc4599b415f320839e36367b273db8501127b38316f3b9f22f17a0b815",
"sha256:8241cac0aae90b82d6b5c443b853723bcc66963970c67e56e71a2609dc4b5eaf",
"sha256:82740818f2f240a5da8dfb8943b360e4f24022b093207160c77cadade47d7c85",
"sha256:8897b7b7ec077c819187a123174b645eb680c13df68354ed99f9b40a50898f77",
"sha256:c2c5250ff0d36fd58550252f54915776940e4e866f38f3a7866d92b32a654b86",
"sha256:ca9f686517ec2c4a4ce930207f75c00bf03d94e5063cbc00a1dc42531511b7eb",
"sha256:d2b3d199647468d410994dbeb8cec5816fb74feb9368aedf300af709ef507e3e",
"sha256:da73d095f8590ad437cd5e9faf6628a218aa7c387e1fdf67b888b47ba56a17f0",
"sha256:e167b6b710c7f7bc54e67ef593f8731e1f45aa35f8a8a7b72d6e42ec76afd4b3",
"sha256:ea634401ca02367c1567f012317502ef3437522e2fc44a3ea1844de028fa4b84",
"sha256:ec6597aa85ce03f3e507566b8bcdf9da2227ec86c4266bd5e6ab4d9e0cc8dab2",
"sha256:f64b232348ee82f13aac22856515ce0195837f6968aeaa94a3d0353ea2ec06a6"
"sha256:06bfafa6e53ccbfb7a94be4687b211a025ce0625e3f3c60bb15cd048a18f3ed8",
"sha256:0db5cf21bd7d092baacb576482b0245102cea2d3cf09f09271ce9f69624ecb6f",
"sha256:125702572be12bcd318e3a14e9e70acd4be69a43664a75f0397e8650fe3c6cc3",
"sha256:1858eff6246bb8bbc080eee78f3dd1528739e3f416cba5f9914e8631b8df9871",
"sha256:315af6268de72bcfa0bb3401350ce7d921f216e6b60de12a363dad128d9d459f",
"sha256:451aaff8b8adf2dd0597cbb1fdcfc8a7d580f33f843b7cce75307a7f20112dd8",
"sha256:58021d6e9b1d88b1105269d0da5e60e778b37dfc0e824efc71343dd003726831",
"sha256:618391152147a1221c87b1b0b7f792cafcfd4b5a685c5c72eeea2ddd29aeceff",
"sha256:6d4daf890e674d191757d8d7d60dc3a29c58c72c7a76a05f1c0a326013f47e8b",
"sha256:74b55f67f4cf026cb84da7a1b04fc2a1d260193d4ad0ea5e9897c8b74c1e76ac",
"sha256:7ceae26f876aabe193b13a0c36d1bb8e3e7e608d17351861b437bd882f617e9f",
"sha256:930b829e8a2abaf43a19f38277ae3c5e1ffcf547b936a927d2587769ae52c296",
"sha256:a18ff4bfa9d64914a84d7b06c46eb86e0cc03113470b3c111255aceb6dcaf81d",
"sha256:ae1cd29fbe6b716855454e44f4bf743465152e15d2d317303fe3b58ee9e5af7a",
"sha256:b1ee5c82cf03b30f6ae4e32d2bcb1e167ef74d6071cbb77c2af30f101d0b360b",
"sha256:bf585476fcbcd37bed08072e8e2db3954ce1bfc68087a2dc9c19cfe0b90979ca",
"sha256:c4a58eeafbd7409054be41a377e726a7904a17c26f45abf18125d21b1215b08b",
"sha256:cce90609e01e1b192fae9e13665058ab46b2ea53a3c05a3ea74a3eb8c3af8857",
"sha256:d610d0ee14dd9109006215c7c0de15eee91230b70a9bce2263461cf7c3720b83",
"sha256:e69a0e36e62279120e648e787b76d79b41e0f9e86c1c636a4f38d415595c722e",
"sha256:f095988548ec5095e3750cdb30e6962273d239b1998ba1aac66c0d5bee7111c1",
"sha256:faf0f5456c059c7b1c29441bdd5e988f0ba75bdc3eea776520d8dcb1e30e1b5c"
],
"version": "==36.0.2"
"version": "==37.0.1"
},
"faker": {
"hashes": [
"sha256:188961065fb5c78ea639f42176f55100f72c90c3a3179ac6c955c4bd712b0511",
"sha256:7758ece2593ce603db117db3d27393c31f4af03f783e176f3f0e14839a4f3426"
"sha256:0d5425894e098410b64aaade38a81074fa30163076251118523adf5bb44f8346",
"sha256:7ab2f741ef1c006ed7274a6ed75695ca8b610f78861566b599ce83c4953bf687"
],
"markers": "python_version >= '3.6'",
"version": "==13.3.4"
"markers": "python_full_version >= '3.6.0'",
"version": "==13.6.0"
},
"ffmpeg-python": {
"hashes": [
@ -296,22 +298,54 @@
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==0.18.2"
},
"google-api-core": {
"hashes": [
"sha256:65480309a7437f739e4476da037af02a3ec8263f1d1f89f72bbdc8f54fe402d2",
"sha256:8fcbe52dc129fd83dca4e638a76f22b3a11579c493e643134e50e9870b233302"
],
"markers": "python_full_version >= '3.6.0'",
"version": "==2.7.2"
},
"google-api-python-client": {
"hashes": [
"sha256:1a7a954fdb778c77ad2a7b28c8e5f4b8761728aca89f5a017e05abd92814b6e9",
"sha256:ba41ce1da337d39d76da1c5affea6e0b95bb6da74aa2adfe5a40618dd67a01fe"
],
"index": "pypi",
"version": "==2.46.0"
},
"google-auth": {
"hashes": [
"sha256:04e224f241c0566477bb35a8a93be8c635210de743bde454d49393cfb605266d",
"sha256:9a88ee548f6fd49467e2e443dfbfe10344e5a270629a137a3a0b3437ec6b02a6"
"sha256:1ba4938e032b73deb51e59c4656a00e0939cf0b1112575099f136babb4563312",
"sha256:349ac49b18b01019453cc99c11c92ed772739778c92f184002b7ab3a5b7ac77d"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==2.6.5"
"version": "==2.6.6"
},
"google-auth-httplib2": {
"hashes": [
"sha256:31e49c36c6b5643b57e82617cb3e021e3e1d2df9da63af67252c02fa9c1f4a10",
"sha256:a07c39fd632becacd3f07718dfd6021bf396978f03ad3ce4321d060015cc30ac"
],
"index": "pypi",
"version": "==0.1.0"
},
"google-auth-oauthlib": {
"hashes": [
"sha256:24f67735513c4c7134dbde2f1dee5a1deb6acc8dfcb577d7bff30d213a28e7b0",
"sha256:30596b824fc6808fdaca2f048e4998cc40fb4b3599eaea66d28dc7085b36c5b8"
],
"markers": "python_version >= '3.6'",
"index": "pypi",
"version": "==0.5.1"
},
"googleapis-common-protos": {
"hashes": [
"sha256:4007500795bcfc269d279f0f7d253ae18d6dc1ff5d5a73613ffe452038b1ec5f",
"sha256:60220c89b8bd5272159bed4929ecdc1243ae1f73437883a499a44a1cbc084086"
],
"markers": "python_full_version >= '3.6.0'",
"version": "==1.56.0"
},
"gspread": {
"hashes": [
"sha256:319766d90db05056293f7ee0ad2b35503a1a40683a75897a2922398cd2016283",
@ -325,9 +359,17 @@
"sha256:70813c1135087a248a4d38cc0e1a0181ffab2188141a93eaf567940c3957ff06",
"sha256:8ddd78563b633ca55346c8cd41ec0af27d3c79931828beffb46ce70a379e7442"
],
"markers": "python_version >= '3.6'",
"markers": "python_full_version >= '3.6.0'",
"version": "==0.13.0"
},
"httplib2": {
"hashes": [
"sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585",
"sha256:8b6a905cb1c79eefd03f8669fd993c36dc341f7c558f056cb5a33b5c2f458543"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==0.20.4"
},
"idna": {
"hashes": [
"sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
@ -497,12 +539,20 @@
"markers": "python_version >= '3.5' and python_version < '4'",
"version": "==1.45.1"
},
"oauth2client": {
"hashes": [
"sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac",
"sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6"
],
"index": "pypi",
"version": "==4.1.3"
},
"oauthlib": {
"hashes": [
"sha256:23a8208d75b902797ea29fd31fa80a15ed9dc2c6c16fe73f5d346f83f6fa27a2",
"sha256:6db33440354787f9b7f3a6dbd4febf5d0f93758354060e802f6c06cb493022fe"
],
"markers": "python_version >= '3.6'",
"markers": "python_full_version >= '3.6.0'",
"version": "==3.2.0"
},
"outcome": {
@ -510,9 +560,39 @@
"sha256:c7dd9375cfd3c12db9801d080a3b63d4b0a261aa996c4c13152380587288d958",
"sha256:e862f01d4e626e63e8f92c38d1f8d5546d3f9cce989263c521b2e7990d186967"
],
"markers": "python_version >= '3.6'",
"markers": "python_full_version >= '3.6.0'",
"version": "==1.1.0"
},
"protobuf": {
"hashes": [
"sha256:06059eb6953ff01e56a25cd02cca1a9649a75a7e65397b5b9b4e929ed71d10cf",
"sha256:097c5d8a9808302fb0da7e20edf0b8d4703274d140fd25c5edabddcde43e081f",
"sha256:284f86a6207c897542d7e956eb243a36bb8f9564c1742b253462386e96c6b78f",
"sha256:32ca378605b41fd180dfe4e14d3226386d8d1b002ab31c969c366549e66a2bb7",
"sha256:3cc797c9d15d7689ed507b165cd05913acb992d78b379f6014e013f9ecb20996",
"sha256:62f1b5c4cd6c5402b4e2d63804ba49a327e0c386c99b1675c8a0fefda23b2067",
"sha256:69ccfdf3657ba59569c64295b7d51325f91af586f8d5793b734260dfe2e94e2c",
"sha256:6f50601512a3d23625d8a85b1638d914a0970f17920ff39cec63aaef80a93fb7",
"sha256:7403941f6d0992d40161aa8bb23e12575637008a5a02283a930addc0508982f9",
"sha256:755f3aee41354ae395e104d62119cb223339a8f3276a0cd009ffabfcdd46bb0c",
"sha256:77053d28427a29987ca9caf7b72ccafee011257561259faba8dd308fda9a8739",
"sha256:7e371f10abe57cee5021797126c93479f59fccc9693dafd6bd5633ab67808a91",
"sha256:9016d01c91e8e625141d24ec1b20fed584703e527d28512aa8c8707f105a683c",
"sha256:9be73ad47579abc26c12024239d3540e6b765182a91dbc88e23658ab71767153",
"sha256:adc31566d027f45efe3f44eeb5b1f329da43891634d61c75a5944e9be6dd42c9",
"sha256:adfc6cf69c7f8c50fd24c793964eef18f0ac321315439d94945820612849c388",
"sha256:af0ebadc74e281a517141daad9d0f2c5d93ab78e9d455113719a45a49da9db4e",
"sha256:cb29edb9eab15742d791e1025dd7b6a8f6fcb53802ad2f6e3adcb102051063ab",
"sha256:cd68be2559e2a3b84f517fb029ee611546f7812b1fdd0aa2ecc9bc6ec0e4fdde",
"sha256:cdee09140e1cd184ba9324ec1df410e7147242b94b5f8b0c64fc89e38a8ba531",
"sha256:db977c4ca738dd9ce508557d4fce0f5aebd105e158c725beec86feb1f6bc20d8",
"sha256:dd5789b2948ca702c17027c84c2accb552fc30f4622a98ab5c51fcfe8c50d3e7",
"sha256:e250a42f15bf9d5b09fe1b293bdba2801cd520a9f5ea2d7fb7536d4441811d20",
"sha256:ff8d8fa42675249bb456f5db06c00de6c2f4c27a065955917b28c4f15978b9c3"
],
"markers": "python_version >= '3.7'",
"version": "==3.20.1"
},
"pyaes": {
"hashes": [
"sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f"
@ -662,7 +742,7 @@
"sha256:5c6bd9dc7a543b7fe4304a631f8a8a3b674e2bbfc49c2ae96200cdbe55df6b17",
"sha256:95c5d300c4e879ee69708c428ba566c59478fd653cc3a22243eeb8ed846950bb"
],
"markers": "python_version >= '3.6' and python_version < '4'",
"markers": "python_version < '4' and python_full_version >= '3.6.0'",
"version": "==4.8"
},
"s3transfer": {
@ -670,7 +750,7 @@
"sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971",
"sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed"
],
"markers": "python_version >= '3.6'",
"markers": "python_full_version >= '3.6.0'",
"version": "==0.5.2"
},
"selenium": {
@ -716,7 +796,7 @@
"sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759",
"sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"
],
"markers": "python_version >= '3.6'",
"markers": "python_full_version >= '3.6.0'",
"version": "==2.3.2.post1"
},
"telethon": {
@ -747,7 +827,19 @@
"markers": "python_version >= '3.5'",
"version": "==0.9.2"
},
"uritemplate": {
"hashes": [
"sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0",
"sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e"
],
"markers": "python_full_version >= '3.6.0'",
"version": "==4.1.1"
},
"urllib3": {
"extras": [
"secure",
"socks"
],
"hashes": [
"sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14",
"sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"
@ -842,5 +934,30 @@
"version": "==3.8.0"
}
},
"develop": {}
"develop": {
"autopep8": {
"hashes": [
"sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979",
"sha256:ed77137193bbac52d029a52c59bec1b0629b5a186c495f1eb21b126ac466083f"
],
"index": "pypi",
"version": "==1.6.0"
},
"pycodestyle": {
"hashes": [
"sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20",
"sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==2.8.0"
},
"toml": {
"hashes": [
"sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
"sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
],
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==0.10.2"
}
}
}

3
README.md
Wyświetl plik

@ -121,7 +121,8 @@ not tested as the API is getting all so far
# TikTok
not tested yet
Always getting invalid URL using the library [https://github.com/msramalho/tiktok-downloader](https://github.com/msramalho/tiktok-downloader) which is a fork of [https://github.com/krypton-byte/tiktok-downloader](https://github.com/krypton-byte/tiktok-downloader) and has a nice test app [https://tkdown.herokuapp.com/](https://tkdown.herokuapp.com/)
# Twitter Video - YoutubeDL

1
abc.txt 100644
Wyświetl plik

@ -0,0 +1 @@
sdf

8
archivers/telegram_archiver.py
Wyświetl plik

@ -11,7 +11,7 @@ from .base_archiver import Archiver, ArchiveResult
class TelegramArchiver(Archiver):
name = "telegram"
def download(self, url, check_if_exists=False, filenumber=""):
def download(self, url, check_if_exists=False, filenumber=None):
# detect URLs that we definitely cannot handle
if 't.me' != self.get_netloc(url):
return False
@ -52,6 +52,10 @@ class TelegramArchiver(Archiver):
video_id = video_url.split('/')[-1].split('?')[0]
key = self.get_key(video_id)
# DM feature flag (not tested as telethon gets all requests)
if filenumber is not None:
key = filenumber + "/" + key
filename = 'tmp/' + key
cdn_url = self.storage.get_cdn_url(key)
@ -81,7 +85,7 @@ class TelegramArchiver(Archiver):
# process thumbnails
key_thumb, thumb_index = self.get_thumbnails(
filename, key, duration=duration)
filename, key, duration=duration, filenumber=filenumber)
os.remove(filename)
return ArchiveResult(status=status, cdn_url=cdn_url, thumbnail=key_thumb, thumbnail_index=thumb_index,

20
archivers/telethon_archiver.py
Wyświetl plik

@ -48,7 +48,7 @@ class TelethonArchiver(Archiver):
return False
status = "success"
screenshot = self.get_screenshot(url)
screenshot = self.get_screenshot(url, filenumber)
# app will ask (stall for user input!) for phone number and auth code if anon.session not found
with self.client.start():
@ -67,6 +67,11 @@ class TelethonArchiver(Archiver):
if len(media_posts) > 1:
key = self.get_html_key(url)
# DM feature flag
if filenumber is not None:
key = filenumber + "/" + key
cdn_url = self.storage.get_cdn_url(key)
if check_if_exists and self.storage.exists(key):
@ -80,26 +85,33 @@ class TelethonArchiver(Archiver):
if len(mp.message) > len(message): message = mp.message
filename = self.client.download_media(mp.media, f'tmp/{chat}_{group_id}/{mp.id}')
key = filename.split('tmp/')[1]
# DM feature flag
if filenumber is not None:
key = filenumber + "/" + key
self.storage.upload(filename, key)
hash = self.get_hash(filename)
cdn_url = self.storage.get_cdn_url(key)
uploaded_media.append({'cdn_url': cdn_url, 'key': key, 'hash': hash})
os.remove(filename)
page_cdn, page_hash, _ = self.generate_media_page_html(url, uploaded_media, html.escape(str(post)))
#DM
page_cdn, page_hash, _ = self.generate_media_page_html(url, uploaded_media, html.escape(str(post)), filenumber=filenumber)
return ArchiveResult(status=status, cdn_url=page_cdn, title=post.message, timestamp=post.date, hash=page_hash, screenshot=screenshot)
elif len(media_posts) == 1:
key = self.get_key(f'{chat}_{post_id}')
filename = self.client.download_media(post.media, f'tmp/{key}')
key = filename.split('tmp/')[1].replace(" ", "")
# DM feature flag
if filenumber is not None:
key = filenumber + "/" + key
self.storage.upload(filename, key)
hash = self.get_hash(filename)
cdn_url = self.storage.get_cdn_url(key)
key_thumb, thumb_index = self.get_thumbnails(filename, key)
key_thumb, thumb_index = self.get_thumbnails(filename, key, filenumber=filenumber)
os.remove(filename)
return ArchiveResult(status=status, cdn_url=cdn_url, title=post.message, thumbnail=key_thumb, thumbnail_index=thumb_index, timestamp=post.date, hash=hash, screenshot=screenshot)
page_cdn, page_hash, _ = self.generate_media_page_html(url, [], html.escape(str(post)))
page_cdn, page_hash, _ = self.generate_media_page_html(url, [], html.escape(str(post)), filenumber=filenumber)
return ArchiveResult(status=status, cdn_url=page_cdn, title=post.message, timestamp=post.date, hash=page_hash, screenshot=screenshot)

5
archivers/tiktok_archiver.py
Wyświetl plik

@ -37,7 +37,7 @@ class TiktokArchiver(Archiver):
self.storage.upload(filename, key)
try:
key_thumb, thumb_index = self.get_thumbnails(filename, key, duration=info.duration)
key_thumb, thumb_index = self.get_thumbnails(filename, key, duration=info.duration, filenumber=filenumber)
except Exception as e:
logger.error(e)
key_thumb = ''
@ -54,8 +54,9 @@ class TiktokArchiver(Archiver):
thumbnail_index=thumb_index, duration=info.duration, title=info.caption, timestamp=info.create.isoformat(),
hash=hash, screenshot=screenshot)
except tiktok_downloader.Except.InvalidUrl:
except tiktok_downloader.Except.InvalidUrl as e:
status = 'Invalid URL'
logger.warning(f'Invalid URL on {url} {e}\n{traceback.format_exc()}')
return ArchiveResult(status=status)
except:

9
auto_archive.py
Wyświetl plik

@ -154,8 +154,13 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES, usefilenumbe
logger.debug(f'Trying {archiver} on row {row}')
try:
# DM . filenumber="" if not to be used
result = archiver.download(url, check_if_exists=True, filenumber=filenumber)
# DM
if usefilenumber:
# using filenumber to store in folders so can't check for existance of that url
result = archiver.download(url, check_if_exists=False, filenumber=filenumber)
else:
result = archiver.download(url, check_if_exists=True, filenumber=filenumber)
except Exception as e:
result = False
logger.error(f'Got unexpected error in row {row} with archiver {archiver} for url {url}: {e}\n{traceback.format_exc()}')

59
dm_drive.py 100644
Wyświetl plik

@ -0,0 +1,59 @@
from __future__ import print_function
import os.path
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
# If modifying these scopes, delete the file token.json.
SCOPES = ['https://www.googleapis.com/auth/drive.metadata.readonly']
def main():
"""Shows basic usage of the Drive v3 API.
Prints the names and ids of the first 10 files the user has access to.
"""
creds = None
# The file token.json stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists('token.json'):
creds = Credentials.from_authorized_user_file('token.json', SCOPES)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
'credentials.json', SCOPES)
creds = flow.run_local_server(port=0)
# Save the credentials for the next run
with open('token.json', 'w') as token:
token.write(creds.to_json())
try:
service = build('drive', 'v3', credentials=creds)
# Call the Drive v3 API
results = service.files().list(
pageSize=10, fields="nextPageToken, files(id, name)").execute()
items = results.get('files', [])
if not items:
print('No files found.')
return
print('Files:')
for item in items:
print(u'{0} ({1})'.format(item['name'], item['id']))
except HttpError as error:
# TODO(developer) - Handle errors from drive API.
print(f'An error occurred: {error}')
if __name__ == '__main__':
main()

34
dm_drive2.py 100644
Wyświetl plik

@ -0,0 +1,34 @@
from google.oauth2 import service_account
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
def main():
# SCOPES = ['https://www.googleapis.com/auth/drive.metadata.readonly']
SCOPES = ['https://www.googleapis.com/auth/drive']
creds = service_account.Credentials.from_service_account_file('service_account.json', scopes=SCOPES)
try:
service = build('drive', 'v3', credentials=creds)
# Call the Drive v3 API
# results = service.files().list(pageSize=20, fields="nextPageToken, files(id, name)").execute()
results = service.files().list().execute()
items = results.get('files', [])
if not items:
print('No files found.')
return
print('Files:')
for item in items:
print(u'{0} ({1})'.format(item['name'], item['id']))
except HttpError as error:
# TODO(developer) - Handle errors from drive API.
print(f'An error occurred: {error}')
if __name__ == '__main__':
main()

82
dm_drive3_upload.py 100644
Wyświetl plik

@ -0,0 +1,82 @@
from __future__ import print_function
import google.auth
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from googleapiclient.http import MediaFileUpload
from google.oauth2 import service_account
def upload_appdata():
# creds, _ = google.auth.default()
SCOPES = ['https://www.googleapis.com/auth/drive']
creds = service_account.Credentials.from_service_account_file('service_account.json', scopes=SCOPES)
try:
# call drive api client
service = build('drive', 'v3', credentials=creds)
# create a file in a new folder
# file_metadata = {
# 'name': 'Invoices',
# 'mimeType': 'application/vnd.google-apps.folder'
# }
# file = service.files().create(body=file_metadata,
# fields='id').execute()
# print('Folder ID: %s' % file.get('id'))
# upload an image
file_metadata = {'name': 'photo.jpg'}
media = MediaFileUpload('files/photo.jpg',
mimetype='image/jpeg')
file = service.files().create(body=file_metadata,
media_body=media,
fields='id').execute()
id = file.get('id')
print(f'id: {id}')
# list files and folders
results = service.files().list().execute()
items = results.get('files', [])
for item in items:
print(u'{0} ({1})'.format(item['name'], item['id']))
# upload an image to a folder
folder_id ='1ljwzoAdKdJMJzRW9gPHDC8fkRykVH83X'
file_metadata = {
'name': 'photo.jpg',
'parents': [folder_id]
}
media = MediaFileUpload('files/photo.jpg',
mimetype='image/jpeg',
resumable=True)
file = service.files().create(body=file_metadata,
media_body=media,
fields='id').execute()
# print 'File ID: %s' % file.get('id')
# pylint: disable=maybe-no-member
# file_metadata = {
# 'name': 'abc.txt',
# 'parents': ['appDataFolder']
# }
# media = MediaFileUpload('abc.txt',
# mimetype='text/txt',
# resumable=True)
# file = service.files().create(body=file_metadata, media_body=media,
# fields='id').execute()
# print(F'File ID: {file.get("id")}')
except HttpError as error:
print(F'An error occurred: {error}')
file = None
return file.get('id')
if __name__ == '__main__':
upload_appdata()

BIN
files/photo.jpg 100644
Wyświetl plik

Plik binarny nie jest wyświetlany.
Side by Side

Po

Szerokość:  |  Wysokość:  |  Rozmiar: 123 KiB