mirror
/
auto-archiver
kopia lustrzana https://github.com/bellingcat/auto-archiver
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

reduce s3 duplicating while keeping random urls via hash (#112)

pull/113/head
Miguel Sozinho Ramalho 2023-12-12 19:12:03 +00:00 zatwierdzone przez GitHub
rodzic 9ee323a654
commit 3e56ef137d
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
9 zmienionych plików z 355 dodań i 294 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

538
Pipfile.lock wygenerowano
Wyświetl plik

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "69db7f6d6f577270b491ade7462bc2f44d5cf39fe1408547647054ee6b69e929" "sha256": "48785b4cb4cf34066866e48f54493b9fb5583cbb42055c402d968d5e80c0c610"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@ -18,96 +18,85 @@
"default": { "default": {
"aiohttp": { "aiohttp": {
"hashes": [ "hashes": [
"sha256:002f23e6ea8d3dd8d149e569fd580c999232b5fbc601c48d55398fbc2e582e8c", "sha256:02ab6006ec3c3463b528374c4cdce86434e7b89ad355e7bf29e2f16b46c7dd6f",
"sha256:01770d8c04bd8db568abb636c1fdd4f7140b284b8b3e0b4584f070180c1e5c62", "sha256:04fa38875e53eb7e354ece1607b1d2fdee2d175ea4e4d745f6ec9f751fe20c7c",
"sha256:0912ed87fee967940aacc5306d3aa8ba3a459fcd12add0b407081fbefc931e53", "sha256:0b0a6a36ed7e164c6df1e18ee47afbd1990ce47cb428739d6c99aaabfaf1b3af",
"sha256:0cccd1de239afa866e4ce5c789b3032442f19c261c7d8a01183fd956b1935349", "sha256:0d406b01a9f5a7e232d1b0d161b40c05275ffbcbd772dc18c1d5a570961a1ca4",
"sha256:0fa375b3d34e71ccccf172cab401cd94a72de7a8cc01847a7b3386204093bb47", "sha256:0e49b08eafa4f5707ecfb321ab9592717a319e37938e301d462f79b4e860c32a",
"sha256:13da35c9ceb847732bf5c6c5781dcf4780e14392e5d3b3c689f6d22f8e15ae31", "sha256:0e7ba7ff228c0d9a2cd66194e90f2bca6e0abca810b786901a569c0de082f489",
"sha256:14cd52ccf40006c7a6cd34a0f8663734e5363fd981807173faf3a017e202fec9", "sha256:11cb254e397a82efb1805d12561e80124928e04e9c4483587ce7390b3866d213",
"sha256:16d330b3b9db87c3883e565340d292638a878236418b23cc8b9b11a054aaa887", "sha256:11ff168d752cb41e8492817e10fb4f85828f6a0142b9726a30c27c35a1835f01",
"sha256:1bed815f3dc3d915c5c1e556c397c8667826fbc1b935d95b0ad680787896a358", "sha256:176df045597e674fa950bf5ae536be85699e04cea68fa3a616cf75e413737eb5",
"sha256:1d84166673694841d8953f0a8d0c90e1087739d24632fe86b1a08819168b4566", "sha256:219a16763dc0294842188ac8a12262b5671817042b35d45e44fd0a697d8c8361",
"sha256:1f13f60d78224f0dace220d8ab4ef1dbc37115eeeab8c06804fec11bec2bbd07", "sha256:22698f01ff5653fe66d16ffb7658f582a0ac084d7da1323e39fd9eab326a1f26",
"sha256:229852e147f44da0241954fc6cb910ba074e597f06789c867cb7fb0621e0ba7a", "sha256:237533179d9747080bcaad4d02083ce295c0d2eab3e9e8ce103411a4312991a0",
"sha256:253bf92b744b3170eb4c4ca2fa58f9c4b87aeb1df42f71d4e78815e6e8b73c9e", "sha256:289ba9ae8e88d0ba16062ecf02dd730b34186ea3b1e7489046fc338bdc3361c4",
"sha256:255ba9d6d5ff1a382bb9a578cd563605aa69bec845680e21c44afc2670607a95", "sha256:2c59e0076ea31c08553e868cec02d22191c086f00b44610f8ab7363a11a5d9d8",
"sha256:2817b2f66ca82ee699acd90e05c95e79bbf1dc986abb62b61ec8aaf851e81c93", "sha256:2c9376e2b09895c8ca8b95362283365eb5c03bdc8428ade80a864160605715f1",
"sha256:2b8d4e166e600dcfbff51919c7a3789ff6ca8b3ecce16e1d9c96d95dd569eb4c", "sha256:3135713c5562731ee18f58d3ad1bf41e1d8883eb68b363f2ffde5b2ea4b84cc7",
"sha256:2d5b785c792802e7b275c420d84f3397668e9d49ab1cb52bd916b3b3ffcf09ad", "sha256:3b9c7426923bb7bd66d409da46c41e3fb40f5caf679da624439b9eba92043fa6",
"sha256:3161ce82ab85acd267c8f4b14aa226047a6bee1e4e6adb74b798bd42c6ae1f80", "sha256:3c0266cd6f005e99f3f51e583012de2778e65af6b73860038b968a0a8888487a",
"sha256:33164093be11fcef3ce2571a0dccd9041c9a93fa3bde86569d7b03120d276c6f", "sha256:41473de252e1797c2d2293804e389a6d6986ef37cbb4a25208de537ae32141dd",
"sha256:39a312d0e991690ccc1a61f1e9e42daa519dcc34ad03eb6f826d94c1190190dd", "sha256:4831df72b053b1eed31eb00a2e1aff6896fb4485301d4ccb208cac264b648db4",
"sha256:3b2ab182fc28e7a81f6c70bfbd829045d9480063f5ab06f6e601a3eddbbd49a0", "sha256:49f0c1b3c2842556e5de35f122fc0f0b721334ceb6e78c3719693364d4af8499",
"sha256:3c68330a59506254b556b99a91857428cab98b2f84061260a67865f7f52899f5", "sha256:4b4c452d0190c5a820d3f5c0f3cd8a28ace48c54053e24da9d6041bf81113183",
"sha256:3f0e27e5b733803333bb2371249f41cf42bae8884863e8e8965ec69bebe53132", "sha256:4ee8caa925aebc1e64e98432d78ea8de67b2272252b0a931d2ac3bd876ad5544",
"sha256:3f5c7ce535a1d2429a634310e308fb7d718905487257060e5d4598e29dc17f0b", "sha256:500f1c59906cd142d452074f3811614be04819a38ae2b3239a48b82649c08821",
"sha256:3fd194939b1f764d6bb05490987bfe104287bbf51b8d862261ccf66f48fb4096", "sha256:5216b6082c624b55cfe79af5d538e499cd5f5b976820eac31951fb4325974501",
"sha256:41bdc2ba359032e36c0e9de5a3bd00d6fb7ea558a6ce6b70acedf0da86458321", "sha256:54311eb54f3a0c45efb9ed0d0a8f43d1bc6060d773f6973efd90037a51cd0a3f",
"sha256:41d55fc043954cddbbd82503d9cc3f4814a40bcef30b3569bc7b5e34130718c1", "sha256:54631fb69a6e44b2ba522f7c22a6fb2667a02fd97d636048478db2fd8c4e98fe",
"sha256:42c89579f82e49db436b69c938ab3e1559e5a4409eb8639eb4143989bc390f2f", "sha256:565760d6812b8d78d416c3c7cfdf5362fbe0d0d25b82fed75d0d29e18d7fc30f",
"sha256:45ad816b2c8e3b60b510f30dbd37fe74fd4a772248a52bb021f6fd65dff809b6", "sha256:598db66eaf2e04aa0c8900a63b0101fdc5e6b8a7ddd805c56d86efb54eb66672",
"sha256:4ac39027011414dbd3d87f7edb31680e1f430834c8cef029f11c66dad0670aa5", "sha256:5c4fa235d534b3547184831c624c0b7c1e262cd1de847d95085ec94c16fddcd5",
"sha256:4d4cbe4ffa9d05f46a28252efc5941e0462792930caa370a6efaf491f412bc66", "sha256:69985d50a2b6f709412d944ffb2e97d0be154ea90600b7a921f95a87d6f108a2",
"sha256:4fcf3eabd3fd1a5e6092d1242295fa37d0354b2eb2077e6eb670accad78e40e1", "sha256:69da0f3ed3496808e8cbc5123a866c41c12c15baaaead96d256477edf168eb57",
"sha256:5d791245a894be071d5ab04bbb4850534261a7d4fd363b094a7b9963e8cdbd31", "sha256:6c93b7c2e52061f0925c3382d5cb8980e40f91c989563d3d32ca280069fd6a87",
"sha256:6c43ecfef7deaf0617cee936836518e7424ee12cb709883f2c9a1adda63cc460", "sha256:70907533db712f7aa791effb38efa96f044ce3d4e850e2d7691abd759f4f0ae0",
"sha256:6c5f938d199a6fdbdc10bbb9447496561c3a9a565b43be564648d81e1102ac22", "sha256:81b77f868814346662c96ab36b875d7814ebf82340d3284a31681085c051320f",
"sha256:6e2f9cc8e5328f829f6e1fb74a0a3a939b14e67e80832975e01929e320386b34", "sha256:82eefaf1a996060602f3cc1112d93ba8b201dbf5d8fd9611227de2003dddb3b7",
"sha256:713103a8bdde61d13490adf47171a1039fd880113981e55401a0f7b42c37d071", "sha256:85c3e3c9cb1d480e0b9a64c658cd66b3cfb8e721636ab8b0e746e2d79a7a9eed",
"sha256:71783b0b6455ac8f34b5ec99d83e686892c50498d5d00b8e56d47f41b38fbe04", "sha256:8a22a34bc594d9d24621091d1b91511001a7eea91d6652ea495ce06e27381f70",
"sha256:76b36b3124f0223903609944a3c8bf28a599b2cc0ce0be60b45211c8e9be97f8", "sha256:8cef8710fb849d97c533f259103f09bac167a008d7131d7b2b0e3a33269185c0",
"sha256:7bc88fc494b1f0311d67f29fee6fd636606f4697e8cc793a2d912ac5b19aa38d", "sha256:8d44e7bf06b0c0a70a20f9100af9fcfd7f6d9d3913e37754c12d424179b4e48f",
"sha256:7ee912f7e78287516df155f69da575a0ba33b02dd7c1d6614dbc9463f43066e3", "sha256:8d7f98fde213f74561be1d6d3fa353656197f75d4edfbb3d94c9eb9b0fc47f5d",
"sha256:86f20cee0f0a317c76573b627b954c412ea766d6ada1a9fcf1b805763ae7feeb", "sha256:8d8e4450e7fe24d86e86b23cc209e0023177b6d59502e33807b732d2deb6975f",
"sha256:89341b2c19fb5eac30c341133ae2cc3544d40d9b1892749cdd25892bbc6ac951", "sha256:8fc49a87ac269d4529da45871e2ffb6874e87779c3d0e2ccd813c0899221239d",
"sha256:8a9b5a0606faca4f6cc0d338359d6fa137104c337f489cd135bb7fbdbccb1e39", "sha256:90ec72d231169b4b8d6085be13023ece8fa9b1bb495e4398d847e25218e0f431",
"sha256:8d399dade330c53b4106160f75f55407e9ae7505263ea86f2ccca6bfcbdb4921", "sha256:91c742ca59045dce7ba76cab6e223e41d2c70d79e82c284a96411f8645e2afff",
"sha256:8e31e9db1bee8b4f407b77fd2507337a0a80665ad7b6c749d08df595d88f1cf5", "sha256:9b05d33ff8e6b269e30a7957bd3244ffbce2a7a35a81b81c382629b80af1a8bf",
"sha256:90c72ebb7cb3a08a7f40061079817133f502a160561d0675b0a6adf231382c92", "sha256:9b05d5cbe9dafcdc733262c3a99ccf63d2f7ce02543620d2bd8db4d4f7a22f83",
"sha256:918810ef188f84152af6b938254911055a72e0f935b5fbc4c1a4ed0b0584aed1", "sha256:9c5857612c9813796960c00767645cb5da815af16dafb32d70c72a8390bbf690",
"sha256:93c15c8e48e5e7b89d5cb4613479d144fda8344e2d886cf694fd36db4cc86865", "sha256:a34086c5cc285be878622e0a6ab897a986a6e8bf5b67ecb377015f06ed316587",
"sha256:96603a562b546632441926cd1293cfcb5b69f0b4159e6077f7c7dbdfb686af4d", "sha256:ab221850108a4a063c5b8a70f00dd7a1975e5a1713f87f4ab26a46e5feac5a0e",
"sha256:99c5ac4ad492b4a19fc132306cd57075c28446ec2ed970973bbf036bcda1bcc6", "sha256:b796b44111f0cab6bbf66214186e44734b5baab949cb5fb56154142a92989aeb",
"sha256:9c19b26acdd08dd239e0d3669a3dddafd600902e37881f13fbd8a53943079dbc", "sha256:b8c3a67eb87394386847d188996920f33b01b32155f0a94f36ca0e0c635bf3e3",
"sha256:9de50a199b7710fa2904be5a4a9b51af587ab24c8e540a7243ab737b45844543", "sha256:bcb6532b9814ea7c5a6a3299747c49de30e84472fa72821b07f5a9818bce0f66",
"sha256:9e2ee0ac5a1f5c7dd3197de309adfb99ac4617ff02b0603fd1e65b07dc772e4b", "sha256:bcc0ea8d5b74a41b621ad4a13d96c36079c81628ccc0b30cfb1603e3dfa3a014",
"sha256:a2ece4af1f3c967a4390c284797ab595a9f1bc1130ef8b01828915a05a6ae684", "sha256:bea94403a21eb94c93386d559bce297381609153e418a3ffc7d6bf772f59cc35",
"sha256:a3628b6c7b880b181a3ae0a0683698513874df63783fd89de99b7b7539e3e8a8", "sha256:bff7e2811814fa2271be95ab6e84c9436d027a0e59665de60edf44e529a42c1f",
"sha256:ad1407db8f2f49329729564f71685557157bfa42b48f4b93e53721a16eb813ed", "sha256:c72444d17777865734aa1a4d167794c34b63e5883abb90356a0364a28904e6c0",
"sha256:b04691bc6601ef47c88f0255043df6f570ada1a9ebef99c34bd0b72866c217ae", "sha256:c7b5d5d64e2a14e35a9240b33b89389e0035e6de8dbb7ffa50d10d8b65c57449",
"sha256:b0cf2a4501bff9330a8a5248b4ce951851e415bdcce9dc158e76cfd55e15085c", "sha256:c7e939f1ae428a86e4abbb9a7c4732bf4706048818dfd979e5e2839ce0159f23",
"sha256:b2fe42e523be344124c6c8ef32a011444e869dc5f883c591ed87f84339de5976", "sha256:c88a15f272a0ad3d7773cf3a37cc7b7d077cbfc8e331675cf1346e849d97a4e5",
"sha256:b30e963f9e0d52c28f284d554a9469af073030030cef8693106d918b2ca92f54", "sha256:c9110c06eaaac7e1f5562caf481f18ccf8f6fdf4c3323feab28a93d34cc646bd",
"sha256:bb54c54510e47a8c7c8e63454a6acc817519337b2b78606c4e840871a3e15349", "sha256:ca7ca5abfbfe8d39e653870fbe8d7710be7a857f8a8386fc9de1aae2e02ce7e4",
"sha256:bd111d7fc5591ddf377a408ed9067045259ff2770f37e2d94e6478d0f3fc0c17", "sha256:cae4c0c2ca800c793cae07ef3d40794625471040a87e1ba392039639ad61ab5b",
"sha256:bdf70bfe5a1414ba9afb9d49f0c912dc524cf60141102f3a11143ba3d291870f", "sha256:cdefe289681507187e375a5064c7599f52c40343a8701761c802c1853a504558",
"sha256:ca80e1b90a05a4f476547f904992ae81eda5c2c85c66ee4195bb8f9c5fb47f28", "sha256:cf2a0ac0615842b849f40c4d7f304986a242f1e68286dbf3bd7a835e4f83acfd",
"sha256:caf486ac1e689dda3502567eb89ffe02876546599bbf915ec94b1fa424eeffd4", "sha256:cfeadf42840c1e870dc2042a232a8748e75a36b52d78968cda6736de55582766",
"sha256:ccc360e87341ad47c777f5723f68adbb52b37ab450c8bc3ca9ca1f3e849e5fe2", "sha256:d737e69d193dac7296365a6dcb73bbbf53bb760ab25a3727716bbd42022e8d7a",
"sha256:d25036d161c4fe2225d1abff2bd52c34ed0b1099f02c208cd34d8c05729882f0", "sha256:d7481f581251bb5558ba9f635db70908819caa221fc79ee52a7f58392778c636",
"sha256:d52d5dc7c6682b720280f9d9db41d36ebe4791622c842e258c9206232251ab2b", "sha256:df9cf74b9bc03d586fc53ba470828d7b77ce51b0582d1d0b5b2fb673c0baa32d",
"sha256:d67f8baed00870aa390ea2590798766256f31dc5ed3ecc737debb6e97e2ede78", "sha256:e1f80197f8b0b846a8d5cf7b7ec6084493950d0882cc5537fb7b96a69e3c8590",
"sha256:d76e8b13161a202d14c9584590c4df4d068c9567c99506497bdd67eaedf36403", "sha256:ecca113f19d5e74048c001934045a2b9368d77b0b17691d905af18bd1c21275e",
"sha256:d95fc1bf33a9a81469aa760617b5971331cdd74370d1214f0b3109272c0e1e3c", "sha256:ee2527134f95e106cc1653e9ac78846f3a2ec1004cf20ef4e02038035a74544d",
"sha256:de6a1c9f6803b90e20869e6b99c2c18cef5cc691363954c93cb9adeb26d9f3ae", "sha256:f27fdaadce22f2ef950fc10dcdf8048407c3b42b73779e48a4e76b3c35bca26c",
"sha256:e1d8cb0b56b3587c5c01de3bf2f600f186da7e7b5f7353d1bf26a8ddca57f965", "sha256:f694dc8a6a3112059258a725a4ebe9acac5fe62f11c77ac4dcf896edfa78ca28",
"sha256:e2a988a0c673c2e12084f5e6ba3392d76c75ddb8ebc6c7e9ead68248101cd446", "sha256:f800164276eec54e0af5c99feb9494c295118fc10a11b997bbb1348ba1a52065",
"sha256:e3f1e3f1a1751bb62b4a1b7f4e435afcdade6c17a4fd9b9d43607cebd242924a", "sha256:ffcd828e37dc219a72c9012ec44ad2e7e3066bec6ff3aaa19e7d435dbf4032ca"
"sha256:e6a00ffcc173e765e200ceefb06399ba09c06db97f401f920513a10c803604ca",
"sha256:e827d48cf802de06d9c935088c2924e3c7e7533377d66b6f31ed175c1620e05e",
"sha256:ebf3fd9f141700b510d4b190094db0ce37ac6361a6806c153c161dc6c041ccda",
"sha256:ec00c3305788e04bf6d29d42e504560e159ccaf0be30c09203b468a6c1ccd3b2",
"sha256:ec4fd86658c6a8964d75426517dc01cbf840bbf32d055ce64a9e63a40fd7b771",
"sha256:efd2fcf7e7b9d7ab16e6b7d54205beded0a9c8566cb30f09c1abe42b4e22bdcb",
"sha256:f0f03211fd14a6a0aed2997d4b1c013d49fb7b50eeb9ffdf5e51f23cfe2c77fa",
"sha256:f628dbf3c91e12f4d6c8b3f092069567d8eb17814aebba3d7d60c149391aee3a",
"sha256:f8ef51e459eb2ad8e7a66c1d6440c808485840ad55ecc3cafefadea47d1b1ba2",
"sha256:fc37e9aef10a696a5a4474802930079ccfc14d9f9c10b4662169671ff034b7df",
"sha256:fdee8405931b0615220e5ddf8cd7edd8592c606a8e4ca2a00704883c396e4479"
], ],
"markers": "python_version >= '3.6'", "markers": "python_version >= '3.8'",
"version": "==3.8.6" "version": "==3.9.1"
}, },
"aiosignal": { "aiosignal": {
"hashes": [ "hashes": [
@ -119,11 +108,11 @@
}, },
"anyio": { "anyio": {
"hashes": [ "hashes": [
"sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f", "sha256:56a415fbc462291813a94528a779597226619c8e78af7de0507333f700011e5f",
"sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a" "sha256:5a0bec7085176715be77df87fc66d6c9d70626bd752fcc85f57cdbee5b3760da"
], ],
"markers": "python_version >= '3.8'", "markers": "python_version >= '3.8'",
"version": "==4.0.0" "version": "==4.1.0"
}, },
"argparse": { "argparse": {
"hashes": [ "hashes": [
@ -138,7 +127,7 @@
"sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f", "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f",
"sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028" "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version < '3.11'",
"version": "==4.0.3" "version": "==4.0.3"
}, },
"attrs": { "attrs": {
@ -151,10 +140,10 @@
}, },
"authlib": { "authlib": {
"hashes": [ "hashes": [
"sha256:2988fdf7d0a5c416f5a37ca4b1e7cee360094940229bc97909aed25880326c72", "sha256:421f7c6b468d907ca2d9afede256f068f87e34d23dd221c07d13d4c234726afb",
"sha256:6de4508ba8125e438a35bcd910d55df7087dccd3dd8517095c2bd9853c372ec1" "sha256:c88984ea00149a90e3537c964327da930779afa4564e354edfd98410bea01911"
], ],
"version": "==0.15.6" "version": "==1.2.1"
}, },
"beautifulsoup4": { "beautifulsoup4": {
"hashes": [ "hashes": [
@ -175,20 +164,20 @@
}, },
"boto3": { "boto3": {
"hashes": [ "hashes": [
"sha256:98b01bbea27740720a06f7c7bc0132ae4ce902e640aab090cfb99ad3278449c3", "sha256:b8c818125489fc0371ef28d806d36d8f1dcb71734fcb0d96b3201563e3e86f22",
"sha256:adfb915958d7b54d876891ea1599dd83189e35a2442eb41ca52b04ea716180b6" "sha256:d02a084b25aa8d46ef917b128e90877efab1ba45f9d1ba3a11f336930378e350"
], ],
"index": "pypi", "index": "pypi",
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==1.28.84" "version": "==1.33.8"
}, },
"botocore": { "botocore": {
"hashes": [ "hashes": [
"sha256:8913bedb96ad0427660dee083aeaa675466eb662bbf1a47781956b5882aadcc5", "sha256:90236e6e69d7e80875d7f9d39383630706edbc1298026698c6c70d9b6a65576e",
"sha256:d65bc05793d1a8a8c191a739f742876b4b403c5c713dc76beef262d18f7984a2" "sha256:e6970bf89cbe2624399aeffce52c253917d8e5a1c671de4054557603ab56c922"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==1.31.84" "version": "==1.33.8"
}, },
"brotli": { "brotli": {
"hashes": [ "hashes": [
@ -276,7 +265,7 @@
"sha256:fd5f17ff8f14003595ab414e45fce13d073e0762394f957182e69035c9f3d7c2", "sha256:fd5f17ff8f14003595ab414e45fce13d073e0762394f957182e69035c9f3d7c2",
"sha256:fdc3ff3bfccdc6b9cc7c342c03aa2400683f0cb891d46e94b64a197910dc4064" "sha256:fdc3ff3bfccdc6b9cc7c342c03aa2400683f0cb891d46e94b64a197910dc4064"
], ],
"markers": "platform_python_implementation >= 'CPython'", "markers": "implementation_name == 'cpython'",
"version": "==1.1.0" "version": "==1.1.0"
}, },
"bs4": { "bs4": {
@ -296,11 +285,11 @@
}, },
"certifi": { "certifi": {
"hashes": [ "hashes": [
"sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1",
"sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"
], ],
"markers": "python_version >= '3.6'", "markers": "python_version >= '3.6'",
"version": "==2023.7.22" "version": "==2023.11.17"
}, },
"cffi": { "cffi": {
"hashes": [ "hashes": [
@ -473,33 +462,33 @@
}, },
"cryptography": { "cryptography": {
"hashes": [ "hashes": [
"sha256:0c327cac00f082013c7c9fb6c46b7cc9fa3c288ca702c74773968173bda421bf", "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960",
"sha256:0d2a6a598847c46e3e321a7aef8af1436f11c27f1254933746304ff014664d84", "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a",
"sha256:227ec057cd32a41c6651701abc0328135e472ed450f47c2766f23267b792a88e", "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc",
"sha256:22892cc830d8b2c89ea60148227631bb96a7da0c1b722f2aac8824b1b7c0b6b8", "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a",
"sha256:392cb88b597247177172e02da6b7a63deeff1937fa6fec3bbf902ebd75d97ec7", "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf",
"sha256:3be3ca726e1572517d2bef99a818378bbcf7d7799d5372a46c79c29eb8d166c1", "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1",
"sha256:573eb7128cbca75f9157dcde974781209463ce56b5804983e11a1c462f0f4e88", "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39",
"sha256:580afc7b7216deeb87a098ef0674d6ee34ab55993140838b14c9b83312b37b86", "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406",
"sha256:5a70187954ba7292c7876734183e810b728b4f3965fbe571421cb2434d279179", "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a",
"sha256:73801ac9736741f220e20435f84ecec75ed70eda90f781a148f1bad546963d81", "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a",
"sha256:7d208c21e47940369accfc9e85f0de7693d9a5d843c2509b3846b2db170dfd20", "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c",
"sha256:8254962e6ba1f4d2090c44daf50a547cd5f0bf446dc658a8e5f8156cae0d8548", "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be",
"sha256:88417bff20162f635f24f849ab182b092697922088b477a7abd6664ddd82291d", "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15",
"sha256:a48e74dad1fb349f3dc1d449ed88e0017d792997a7ad2ec9587ed17405667e6d", "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2",
"sha256:b948e09fe5fb18517d99994184854ebd50b57248736fd4c720ad540560174ec5", "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d",
"sha256:c707f7afd813478e2019ae32a7c49cd932dd60ab2d2a93e796f68236b7e1fbf1", "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157",
"sha256:d38e6031e113b7421db1de0c1b1f7739564a88f1684c6b89234fbf6c11b75147", "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003",
"sha256:d3977f0e276f6f5bf245c403156673db103283266601405376f075c849a0b936", "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248",
"sha256:da6a0ff8f1016ccc7477e6339e1d50ce5f59b88905585f77193ebd5068f1e797", "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a",
"sha256:e270c04f4d9b5671ebcc792b3ba5d4488bf7c42c3c241a3748e2599776f29696", "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec",
"sha256:e886098619d3815e0ad5790c973afeee2c0e6e04b4da90b88e6bd06e2a0b1b72", "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309",
"sha256:ec3b055ff8f1dce8e6ef28f626e0972981475173d7973d63f271b29c8a2897da", "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7",
"sha256:fba1e91467c65fe64a82c689dc6cf58151158993b13eb7a7f3f4b7f395636723" "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"
], ],
"index": "pypi", "index": "pypi",
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==41.0.5" "version": "==41.0.7"
}, },
"dataclasses-json": { "dataclasses-json": {
"hashes": [ "hashes": [
@ -512,20 +501,20 @@
}, },
"dateparser": { "dateparser": {
"hashes": [ "hashes": [
"sha256:070b29b5bbf4b1ec2cd51c96ea040dc68a614de703910a91ad1abba18f9f379f", "sha256:0b21ad96534e562920a0083e97fd45fa959882d4162acc358705144520a35830",
"sha256:86b8b7517efcc558f085a142cdb7620f0921543fcabdb538c8a4c4001d8178e3" "sha256:7975b43a4222283e0ae15be7b4999d08c9a70e2d378ac87385b1ccf2cffbbb30"
], ],
"index": "pypi", "index": "pypi",
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==1.1.8" "version": "==1.2.0"
}, },
"exceptiongroup": { "exceptiongroup": {
"hashes": [ "hashes": [
"sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9", "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14",
"sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3" "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"
], ],
"markers": "python_version < '3.11'", "markers": "python_version < '3.11'",
"version": "==1.1.3" "version": "==1.2.0"
}, },
"ffmpeg-python": { "ffmpeg-python": {
"hashes": [ "hashes": [
@ -635,20 +624,20 @@
}, },
"google-api-python-client": { "google-api-python-client": {
"hashes": [ "hashes": [
"sha256:51d7bf676f41a77b00b7b9c72ace0c1db3dd5a4dd392a13ae897cf4f571a3539", "sha256:72e7d46cc70908d808e29f16d983b441783fe56b694cec132db9af9fb991daa2",
"sha256:ef6d4c1a17fe9ec0894fc6d4f61e751c4b859fb33f2ab5b881ceb0b80ba442ba" "sha256:d06390c25477c361d52639fe00ef912c3fab8dafc7fbf29580c1144e92523a79"
], ],
"index": "pypi", "index": "pypi",
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==2.107.0" "version": "==2.109.0"
}, },
"google-auth": { "google-auth": {
"hashes": [ "hashes": [
"sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3", "sha256:2e7d37702e617ea9f4ac2568625a9fc2cdea910dcdca90e378db09ac94ee9ab3",
"sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2" "sha256:cabc5c6fd327d71baa820f745118ecb2b709412d153419a43173d91495ef8715"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==2.23.4" "version": "==2.25.0"
}, },
"google-auth-httplib2": { "google-auth-httplib2": {
"hashes": [ "hashes": [
@ -677,12 +666,12 @@
}, },
"gspread": { "gspread": {
"hashes": [ "hashes": [
"sha256:6fc8e0e627ec878b57caa69ce1c86225b54bee073bbfe5252f3868ea9a61d965", "sha256:4241af683242d5bcb87d611044a96ba47a572815b622613a0050cec48bb5b78e",
"sha256:a3fb9bdf1c185aeee32a6cf533b32628012f065327c50a810f1c909dc5eba64b" "sha256:e14e4392ef380fe9c6a768b971de4d9451a9a707904208e91c4fddd4228c6cc5"
], ],
"index": "pypi", "index": "pypi",
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==5.12.0" "version": "==5.12.2"
}, },
"h11": { "h11": {
"hashes": [ "hashes": [
@ -710,19 +699,19 @@
}, },
"httpx": { "httpx": {
"hashes": [ "hashes": [
"sha256:fec7d6cc5c27c578a391f7e87b9aa7d3d8fbcd034f6399f9f79b45bcc12a866a", "sha256:8b8fcaa0c8ea7b05edd69a094e63a2094c4efcb48129fb757361bc423c0ad9e8",
"sha256:ffd96d5cf901e63863d9f1b4b6807861dbea4d301613415d9e6e57ead15fc5d0" "sha256:a05d3d052d9b2dfce0e3896636467f8a5342fb2b902c819428e1ac65413ca118"
], ],
"markers": "python_version >= '3.8'", "markers": "python_version >= '3.8'",
"version": "==0.25.1" "version": "==0.25.2"
}, },
"idna": { "idna": {
"hashes": [ "hashes": [
"sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca",
"sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"
], ],
"markers": "python_version >= '3.5'", "markers": "python_version >= '3.5'",
"version": "==3.4" "version": "==3.6"
}, },
"instaloader": { "instaloader": {
"hashes": [ "hashes": [
@ -757,6 +746,15 @@
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==1.0.1" "version": "==1.0.1"
}, },
"jsonlines": {
"hashes": [
"sha256:0c6d2c09117550c089995247f605ae4cf77dd1533041d366351f6f298822ea74",
"sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55"
],
"index": "pypi",
"markers": "python_version >= '3.8'",
"version": "==4.0.0"
},
"loguru": { "loguru": {
"hashes": [ "hashes": [
"sha256:003d71e3d3ed35f0f8984898359d65b79e5b21943f78af86aa5491210429b8eb", "sha256:003d71e3d3ed35f0f8984898359d65b79e5b21943f78af86aa5491210429b8eb",
@ -1203,20 +1201,20 @@
}, },
"protobuf": { "protobuf": {
"hashes": [ "hashes": [
"sha256:1a3ba712877e6d37013cdc3476040ea1e313a6c2e1580836a94f76b3c176d575", "sha256:0bf384e75b92c42830c0a679b0cd4d6e2b36ae0cf3dbb1e1dfdda48a244f4bcd",
"sha256:1a53d6f64b00eecf53b65ff4a8c23dc95df1fa1e97bb06b8122e5a64f49fc90a", "sha256:0f881b589ff449bf0b931a711926e9ddaad3b35089cc039ce1af50b21a4ae8cb",
"sha256:32ac2100b0e23412413d948c03060184d34a7c50b3e5d7524ee96ac2b10acf51", "sha256:1484f9e692091450e7edf418c939e15bfc8fc68856e36ce399aed6889dae8bb0",
"sha256:5c1203ac9f50e4853b0a0bfffd32c67118ef552a33942982eeab543f5c634395", "sha256:193f50a6ab78a970c9b4f148e7c750cfde64f59815e86f686c22e26b4fe01ce7",
"sha256:63714e79b761a37048c9701a37438aa29945cd2417a97076048232c1df07b701", "sha256:3497c1af9f2526962f09329fd61a36566305e6c72da2590ae0d7d1322818843b",
"sha256:683dc44c61f2620b32ce4927de2108f3ebe8ccf2fd716e1e684e5a50da154054", "sha256:57d65074b4f5baa4ab5da1605c02be90ac20c8b40fb137d6a8df9f416b0d0ce2",
"sha256:68f7caf0d4f012fd194a301420cf6aa258366144d814f358c5b32558228afa7c", "sha256:8bdbeaddaac52d15c6dce38c71b03038ef7772b977847eb6d374fc86636fa510",
"sha256:b2cf8b5d381f9378afe84618288b239e75665fe58d0f3fd5db400959274296e9", "sha256:a19731d5e83ae4737bb2a089605e636077ac001d18781b3cf489b9546c7c80d6",
"sha256:c40ff8f00aa737938c5378d461637d15c442a12275a81019cc2fef06d81c9419", "sha256:abc0525ae2689a8000837729eef7883b9391cd6aa7950249dcf5a4ede230d5dd",
"sha256:cf21faba64cd2c9a3ed92b7a67f226296b10159dbb8fbc5e854fc90657d908e4", "sha256:becc576b7e6b553d22cbdf418686ee4daa443d7217999125c045ad56322dda10",
"sha256:d94a33db8b7ddbd0af7c467475fb9fde0c705fb315a8433c0e2020942b863a1f" "sha256:ca37bf6a6d0046272c152eea90d2e4ef34593aaa32e8873fc14c16440f22d4b7"
], ],
"markers": "python_version >= '3.8'", "markers": "python_version >= '3.8'",
"version": "==4.25.0" "version": "==4.25.1"
}, },
"pyaes": { "pyaes": {
"hashes": [ "hashes": [
@ -1226,11 +1224,11 @@
}, },
"pyasn1": { "pyasn1": {
"hashes": [ "hashes": [
"sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57", "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58",
"sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde" "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==0.5.0" "version": "==0.5.1"
}, },
"pyasn1-modules": { "pyasn1-modules": {
"hashes": [ "hashes": [
@ -1287,11 +1285,11 @@
}, },
"pygments": { "pygments": {
"hashes": [ "hashes": [
"sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692", "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
"sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29" "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==2.16.1" "version": "==2.17.2"
}, },
"pyparsing": { "pyparsing": {
"hashes": [ "hashes": [
@ -1328,12 +1326,12 @@
}, },
"python-twitter-v2": { "python-twitter-v2": {
"hashes": [ "hashes": [
"sha256:2397d518c17bfbc16a3d414b1cf6d3c231fd8d322f21c755ac2215c9ee675537", "sha256:1ad9cc95ef50d3c4ace8c81296ae9940db405120d3702b24b7ed5db46df58be6",
"sha256:4e03a30b2570fa4f17fbc7293d850fb8276c66be106d55e460b9287de37e1dd2" "sha256:9e2e1949cad43485ba5ce80f4464ea3dcfc35c081e69c4b1b8a0784a48aa2f85"
], ],
"index": "pypi", "index": "pypi",
"markers": "python_version >= '3.6' and python_version < '4.0'", "markers": "python_version >= '3.7' and python_version < '4.0'",
"version": "==0.8.1" "version": "==0.9.0"
}, },
"pytz": { "pytz": {
"hashes": [ "hashes": [
@ -1522,11 +1520,11 @@
}, },
"rich": { "rich": {
"hashes": [ "hashes": [
"sha256:2b38e2fe9ca72c9a00170a1a2d20c63c790d0e10ef1fe35eba76e1e7b1d7d245", "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa",
"sha256:5c14d22737e6d5084ef4771b62d5d4363165b403455a30a1c8ca39dc7b644bef" "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_full_version >= '3.7.0'",
"version": "==13.6.0" "version": "==13.7.0"
}, },
"rsa": { "rsa": {
"hashes": [ "hashes": [
@ -1538,11 +1536,11 @@
}, },
"s3transfer": { "s3transfer": {
"hashes": [ "hashes": [
"sha256:10d6923c6359175f264811ef4bf6161a3156ce8e350e705396a7557d6293c33a", "sha256:368ac6876a9e9ed91f6bc86581e319be08188dc60d50e0d56308ed5765446283",
"sha256:fd3889a66f5fe17299fe75b82eae6cf722554edca744ca5d5fe308b104883d2e" "sha256:c9e56cbe88b28d8e197cf841f1f0c130f246595e77ae5b5a05b69fe7cb83de76"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==0.7.0" "version": "==0.8.2"
}, },
"selenium": { "selenium": {
"hashes": [ "hashes": [
@ -1595,11 +1593,11 @@
}, },
"telethon": { "telethon": {
"hashes": [ "hashes": [
"sha256:9896c0d7386121e8e67cf9a20c3aa4743e33cb24253f924e2fc1397df0281890" "sha256:9e515bac70fc5bc58cbace0e193a42bb42511f16bb7a319707f860476e8ce164"
], ],
"index": "pypi", "index": "pypi",
"markers": "python_version >= '3.5'", "markers": "python_version >= '3.5'",
"version": "==1.32.1" "version": "==1.33.0"
}, },
"text-unidecode": { "text-unidecode": {
"hashes": [ "hashes": [
@ -1676,7 +1674,7 @@
"sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84",
"sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e" "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "markers": "python_version >= '3.7'",
"version": "==2.0.7" "version": "==2.0.7"
}, },
"vk-api": { "vk-api": {
@ -1778,7 +1776,7 @@
"sha256:fc4e7fa5414512b481a2483775a8e8be7803a35b30ca805afa4998a84f9fd9e8", "sha256:fc4e7fa5414512b481a2483775a8e8be7803a35b30ca805afa4998a84f9fd9e8",
"sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7" "sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.8'",
"version": "==12.0" "version": "==12.0"
}, },
"werkzeug": { "werkzeug": {
@ -1799,92 +1797,108 @@
}, },
"yarl": { "yarl": {
"hashes": [ "hashes": [
"sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571", "sha256:09c19e5f4404574fcfb736efecf75844ffe8610606f3fccc35a1515b8b6712c4",
"sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3", "sha256:0ab5baaea8450f4a3e241ef17e3d129b2143e38a685036b075976b9c415ea3eb",
"sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3", "sha256:0d155a092bf0ebf4a9f6f3b7a650dc5d9a5bbb585ef83a52ed36ba46f55cc39d",
"sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c", "sha256:126638ab961633f0940a06e1c9d59919003ef212a15869708dcb7305f91a6732",
"sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7", "sha256:1a0a4f3aaa18580038cfa52a7183c8ffbbe7d727fe581300817efc1e96d1b0e9",
"sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04", "sha256:1d93461e2cf76c4796355494f15ffcb50a3c198cc2d601ad8d6a96219a10c363",
"sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191", "sha256:26a1a8443091c7fbc17b84a0d9f38de34b8423b459fb853e6c8cdfab0eacf613",
"sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea", "sha256:271d63396460b6607b588555ea27a1a02b717ca2e3f2cf53bdde4013d7790929",
"sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4", "sha256:28a108cb92ce6cf867690a962372996ca332d8cda0210c5ad487fe996e76b8bb",
"sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4", "sha256:29beac86f33d6c7ab1d79bd0213aa7aed2d2f555386856bb3056d5fdd9dab279",
"sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095", "sha256:2c757f64afe53a422e45e3e399e1e3cf82b7a2f244796ce80d8ca53e16a49b9f",
"sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e", "sha256:2dad8166d41ebd1f76ce107cf6a31e39801aee3844a54a90af23278b072f1ccf",
"sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74", "sha256:2dc72e891672343b99db6d497024bf8b985537ad6c393359dc5227ef653b2f17",
"sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef", "sha256:2f3c8822bc8fb4a347a192dd6a28a25d7f0ea3262e826d7d4ef9cc99cd06d07e",
"sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33", "sha256:32435d134414e01d937cd9d6cc56e8413a8d4741dea36af5840c7750f04d16ab",
"sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde", "sha256:3cfa4dbe17b2e6fca1414e9c3bcc216f6930cb18ea7646e7d0d52792ac196808",
"sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45", "sha256:3d5434b34100b504aabae75f0622ebb85defffe7b64ad8f52b8b30ec6ef6e4b9",
"sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf", "sha256:4003f380dac50328c85e85416aca6985536812c082387255c35292cb4b41707e",
"sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b", "sha256:44e91a669c43f03964f672c5a234ae0d7a4d49c9b85d1baa93dec28afa28ffbd",
"sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac", "sha256:4a14907b597ec55740f63e52d7fee0e9ee09d5b9d57a4f399a7423268e457b57",
"sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0", "sha256:4ce77d289f8d40905c054b63f29851ecbfd026ef4ba5c371a158cfe6f623663e",
"sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528", "sha256:4d6d74a97e898c1c2df80339aa423234ad9ea2052f66366cef1e80448798c13d",
"sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716", "sha256:51382c72dd5377861b573bd55dcf680df54cea84147c8648b15ac507fbef984d",
"sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb", "sha256:525cd69eff44833b01f8ef39aa33a9cc53a99ff7f9d76a6ef6a9fb758f54d0ff",
"sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18", "sha256:53ec65f7eee8655bebb1f6f1607760d123c3c115a324b443df4f916383482a67",
"sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72", "sha256:5f74b015c99a5eac5ae589de27a1201418a5d9d460e89ccb3366015c6153e60a",
"sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6", "sha256:6280353940f7e5e2efaaabd686193e61351e966cc02f401761c4d87f48c89ea4",
"sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582", "sha256:632c7aeb99df718765adf58eacb9acb9cbc555e075da849c1378ef4d18bf536a",
"sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5", "sha256:6465d36381af057d0fab4e0f24ef0e80ba61f03fe43e6eeccbe0056e74aadc70",
"sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368", "sha256:66a6dbf6ca7d2db03cc61cafe1ee6be838ce0fbc97781881a22a58a7c5efef42",
"sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc", "sha256:6d350388ba1129bc867c6af1cd17da2b197dff0d2801036d2d7d83c2d771a682",
"sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9", "sha256:7217234b10c64b52cc39a8d82550342ae2e45be34f5bff02b890b8c452eb48d7",
"sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be", "sha256:721ee3fc292f0d069a04016ef2c3a25595d48c5b8ddc6029be46f6158d129c92",
"sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a", "sha256:72a57b41a0920b9a220125081c1e191b88a4cdec13bf9d0649e382a822705c65",
"sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80", "sha256:73cc83f918b69110813a7d95024266072d987b903a623ecae673d1e71579d566",
"sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8", "sha256:778df71c8d0c8c9f1b378624b26431ca80041660d7be7c3f724b2c7a6e65d0d6",
"sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6", "sha256:79e1df60f7c2b148722fb6cafebffe1acd95fd8b5fd77795f56247edaf326752",
"sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417", "sha256:7c86d0d0919952d05df880a1889a4f0aeb6868e98961c090e335671dea5c0361",
"sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574", "sha256:7eaf13af79950142ab2bbb8362f8d8d935be9aaf8df1df89c86c3231e4ff238a",
"sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59", "sha256:828235a2a169160ee73a2fcfb8a000709edf09d7511fccf203465c3d5acc59e4",
"sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608", "sha256:8535e111a064f3bdd94c0ed443105934d6f005adad68dd13ce50a488a0ad1bf3",
"sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82", "sha256:88d2c3cc4b2f46d1ba73d81c51ec0e486f59cc51165ea4f789677f91a303a9a7",
"sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1", "sha256:8a2538806be846ea25e90c28786136932ec385c7ff3bc1148e45125984783dc6",
"sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3", "sha256:8dab30b21bd6fb17c3f4684868c7e6a9e8468078db00f599fb1c14e324b10fca",
"sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d", "sha256:8f18a7832ff85dfcd77871fe677b169b1bc60c021978c90c3bb14f727596e0ae",
"sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8", "sha256:946db4511b2d815979d733ac6a961f47e20a29c297be0d55b6d4b77ee4b298f6",
"sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc", "sha256:96758e56dceb8a70f8a5cff1e452daaeff07d1cc9f11e9b0c951330f0a2396a7",
"sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac", "sha256:9a172c3d5447b7da1680a1a2d6ecdf6f87a319d21d52729f45ec938a7006d5d8",
"sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8", "sha256:9a5211de242754b5e612557bca701f39f8b1a9408dff73c6db623f22d20f470e",
"sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955", "sha256:9df9a0d4c5624790a0dea2e02e3b1b3c69aed14bcb8650e19606d9df3719e87d",
"sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0", "sha256:aa4643635f26052401750bd54db911b6342eb1a9ac3e74f0f8b58a25d61dfe41",
"sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367", "sha256:aed37db837ecb5962469fad448aaae0f0ee94ffce2062cf2eb9aed13328b5196",
"sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb", "sha256:af52725c7c39b0ee655befbbab5b9a1b209e01bb39128dce0db226a10014aacc",
"sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a", "sha256:b0b8c06afcf2bac5a50b37f64efbde978b7f9dc88842ce9729c020dc71fae4ce",
"sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623", "sha256:b61e64b06c3640feab73fa4ff9cb64bd8182de52e5dc13038e01cfe674ebc321",
"sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2", "sha256:b7831566595fe88ba17ea80e4b61c0eb599f84c85acaa14bf04dd90319a45b90",
"sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6", "sha256:b8bc5b87a65a4e64bc83385c05145ea901b613d0d3a434d434b55511b6ab0067",
"sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7", "sha256:b8d51817cf4b8d545963ec65ff06c1b92e5765aa98831678d0e2240b6e9fd281",
"sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4", "sha256:b9f9cafaf031c34d95c1528c16b2fa07b710e6056b3c4e2e34e9317072da5d1a",
"sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051", "sha256:bb72d2a94481e7dc7a0c522673db288f31849800d6ce2435317376a345728225",
"sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938", "sha256:c25ec06e4241e162f5d1f57c370f4078797ade95c9208bd0c60f484834f09c96",
"sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8", "sha256:c405d482c320a88ab53dcbd98d6d6f32ada074f2d965d6e9bf2d823158fa97de",
"sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9", "sha256:c4472fe53ebf541113e533971bd8c32728debc4c6d8cc177f2bff31d011ec17e",
"sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3", "sha256:c4b1efb11a8acd13246ffb0bee888dd0e8eb057f8bf30112e3e21e421eb82d4a",
"sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5", "sha256:c5f3faeb8100a43adf3e7925d556801d14b5816a0ac9e75e22948e787feec642",
"sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9", "sha256:c6f034386e5550b5dc8ded90b5e2ff7db21f0f5c7de37b6efc5dac046eb19c10",
"sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333", "sha256:c99ddaddb2fbe04953b84d1651149a0d85214780e4d0ee824e610ab549d98d92",
"sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185", "sha256:ca6b66f69e30f6e180d52f14d91ac854b8119553b524e0e28d5291a724f0f423",
"sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3", "sha256:cccdc02e46d2bd7cb5f38f8cc3d9db0d24951abd082b2f242c9e9f59c0ab2af3",
"sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560", "sha256:cd49a908cb6d387fc26acee8b7d9fcc9bbf8e1aca890c0b2fdfd706057546080",
"sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b", "sha256:cf7a4e8de7f1092829caef66fd90eaf3710bc5efd322a816d5677b7664893c93",
"sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7", "sha256:cfd77e8e5cafba3fb584e0f4b935a59216f352b73d4987be3af51f43a862c403",
"sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78", "sha256:d34c4f80956227f2686ddea5b3585e109c2733e2d4ef12eb1b8b4e84f09a2ab6",
"sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7" "sha256:d61a0ca95503867d4d627517bcfdc28a8468c3f1b0b06c626f30dd759d3999fd",
"sha256:d81657b23e0edb84b37167e98aefb04ae16cbc5352770057893bd222cdc6e45f",
"sha256:d92d897cb4b4bf915fbeb5e604c7911021a8456f0964f3b8ebbe7f9188b9eabb",
"sha256:dd318e6b75ca80bff0b22b302f83a8ee41c62b8ac662ddb49f67ec97e799885d",
"sha256:dd952b9c64f3b21aedd09b8fe958e4931864dba69926d8a90c90d36ac4e28c9a",
"sha256:e0e7e83f31e23c5d00ff618045ddc5e916f9e613d33c5a5823bc0b0a0feb522f",
"sha256:e0f17d1df951336a02afc8270c03c0c6e60d1f9996fcbd43a4ce6be81de0bd9d",
"sha256:e2a16ef5fa2382af83bef4a18c1b3bcb4284c4732906aa69422cf09df9c59f1f",
"sha256:e36021db54b8a0475805acc1d6c4bca5d9f52c3825ad29ae2d398a9d530ddb88",
"sha256:e73db54c967eb75037c178a54445c5a4e7461b5203b27c45ef656a81787c0c1b",
"sha256:e741bd48e6a417bdfbae02e088f60018286d6c141639359fb8df017a3b69415a",
"sha256:f7271d6bd8838c49ba8ae647fc06469137e1c161a7ef97d778b72904d9b68696",
"sha256:fc391e3941045fd0987c77484b2799adffd08e4b6735c4ee5f054366a2e1551d",
"sha256:fc94441bcf9cb8c59f51f23193316afefbf3ff858460cb47b5758bf66a14d130",
"sha256:fe34befb8c765b8ce562f0200afda3578f8abb159c76de3ab354c80b72244c41",
"sha256:fe8080b4f25dfc44a86bedd14bc4f9d469dfc6456e6f3c5d9077e81a5fedfba7",
"sha256:ff34cb09a332832d1cf38acd0f604c068665192c6107a439a92abfd8acf90fe2"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==1.9.2" "version": "==1.9.3"
}, },
"yt-dlp": { "yt-dlp": {
"hashes": [ "hashes": [
"sha256:2b069f22675532eebacdfd6372b1825651a751fef848de9ae6efe6491b2dc38a", "sha256:0322ba85aa4afdb75f8641ed550e5958964daff034aeb477abb15031fd9a51ed",
"sha256:e026ea1c435ff36eef1215bc4c5bb8c479938b90054997ba99f63a4541fe63b4" "sha256:f0ccdaf12e08b15902601a4671c7ab12906d7b11de3ae75fa6506811c24ec5da"
], ],
"index": "pypi", "index": "pypi",
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==2023.10.13" "version": "==2023.11.16"
} }
}, },
"develop": { "develop": {

6
src/auto_archiver/archivers/tiktok_archiver.py
Wyświetl plik

@ -1,9 +1,11 @@
import json, os, traceback, uuid import json, os, traceback
import tiktok_downloader import tiktok_downloader
from loguru import logger from loguru import logger
from . import Archiver from . import Archiver
from ..core import Metadata, Media, ArchivingContext from ..core import Metadata, Media, ArchivingContext
from ..utils.misc import random_str
class TiktokArchiver(Archiver): class TiktokArchiver(Archiver):
@ -37,7 +39,7 @@ class TiktokArchiver(Archiver):
logger.warning(f'Other Tiktok error {error}') logger.warning(f'Other Tiktok error {error}')
try: try:
filename = os.path.join(ArchivingContext.get_tmp_dir(), f'{str(uuid.uuid4())[0:8]}.mp4') filename = os.path.join(ArchivingContext.get_tmp_dir(), f'{random_str(8)}.mp4')
tiktok_media = tiktok_downloader.snaptik(url).get_media() tiktok_media = tiktok_downloader.snaptik(url).get_media()
if len(tiktok_media) <= 0: if len(tiktok_media) <= 0:

7
src/auto_archiver/enrichers/screenshot_enricher.py
Wyświetl plik

@ -1,9 +1,10 @@
from loguru import logger from loguru import logger
import time, uuid, os import time, os
from selenium.common.exceptions import TimeoutException from selenium.common.exceptions import TimeoutException
from . import Enricher from . import Enricher
from ..utils import Webdriver, UrlUtil from ..utils import Webdriver, UrlUtil, random_str
from ..core import Media, Metadata, ArchivingContext from ..core import Media, Metadata, ArchivingContext
class ScreenshotEnricher(Enricher): class ScreenshotEnricher(Enricher):
@ -29,7 +30,7 @@ class ScreenshotEnricher(Enricher):
try: try:
driver.get(url) driver.get(url)
time.sleep(int(self.sleep_before_screenshot)) time.sleep(int(self.sleep_before_screenshot))
screenshot_file = os.path.join(ArchivingContext.get_tmp_dir(), f"screenshot_{str(uuid.uuid4())[0:8]}.png") screenshot_file = os.path.join(ArchivingContext.get_tmp_dir(), f"screenshot_{random_str(8)}.png")
driver.save_screenshot(screenshot_file) driver.save_screenshot(screenshot_file)
to_enrich.add_media(Media(filename=screenshot_file), id="screenshot") to_enrich.add_media(Media(filename=screenshot_file), id="screenshot")
except TimeoutException: except TimeoutException:

5
src/auto_archiver/enrichers/thumbnail_enricher.py
Wyświetl plik

@ -1,8 +1,9 @@
import ffmpeg, os, uuid import ffmpeg, os
from loguru import logger from loguru import logger
from . import Enricher from . import Enricher
from ..core import Media, Metadata, ArchivingContext from ..core import Media, Metadata, ArchivingContext
from ..utils.misc import random_str
class ThumbnailEnricher(Enricher): class ThumbnailEnricher(Enricher):
@ -23,7 +24,7 @@ class ThumbnailEnricher(Enricher):
logger.debug(f"generating thumbnails") logger.debug(f"generating thumbnails")
for i, m in enumerate(to_enrich.media[::]): for i, m in enumerate(to_enrich.media[::]):
if m.is_video(): if m.is_video():
folder = os.path.join(ArchivingContext.get_tmp_dir(), str(uuid.uuid4())) folder = os.path.join(ArchivingContext.get_tmp_dir(), random_str(24))
os.makedirs(folder, exist_ok=True) os.makedirs(folder, exist_ok=True)
logger.debug(f"generating thumbnails for {m.filename}") logger.debug(f"generating thumbnails for {m.filename}")
fps, duration = 0.5, m.get("duration") fps, duration = 0.5, m.get("duration")

6
src/auto_archiver/enrichers/wacz_enricher.py
Wyświetl plik

@ -1,6 +1,6 @@
import jsonlines import jsonlines
import mimetypes import mimetypes
import os, shutil, subprocess, uuid import os, shutil, subprocess
from zipfile import ZipFile from zipfile import ZipFile
from loguru import logger from loguru import logger
from warcio.archiveiterator import ArchiveIterator from warcio.archiveiterator import ArchiveIterator
@ -8,7 +8,7 @@ from warcio.archiveiterator import ArchiveIterator
from ..core import Media, Metadata, ArchivingContext from ..core import Media, Metadata, ArchivingContext
from . import Enricher from . import Enricher
from ..archivers import Archiver from ..archivers import Archiver
from ..utils import UrlUtil from ..utils import UrlUtil, random_str
class WaczArchiverEnricher(Enricher, Archiver): class WaczArchiverEnricher(Enricher, Archiver):
@ -47,7 +47,7 @@ class WaczArchiverEnricher(Enricher, Archiver):
url = to_enrich.get_url() url = to_enrich.get_url()
collection = str(uuid.uuid4())[0:8] collection = random_str(8)
browsertrix_home_host = os.environ.get('BROWSERTRIX_HOME_HOST') or os.path.abspath(ArchivingContext.get_tmp_dir()) browsertrix_home_host = os.environ.get('BROWSERTRIX_HOME_HOST') or os.path.abspath(ArchivingContext.get_tmp_dir())
browsertrix_home_container = os.environ.get('BROWSERTRIX_HOME_CONTAINER') or browsertrix_home_host browsertrix_home_container = os.environ.get('BROWSERTRIX_HOME_CONTAINER') or browsertrix_home_host

5
src/auto_archiver/formatters/html_formatter.py
Wyświetl plik

@ -1,6 +1,6 @@
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass from dataclasses import dataclass
import mimetypes, uuid, os, pathlib import mimetypes, os, pathlib
from jinja2 import Environment, FileSystemLoader from jinja2 import Environment, FileSystemLoader
from urllib.parse import quote from urllib.parse import quote
from loguru import logger from loguru import logger
@ -9,6 +9,7 @@ from ..version import __version__
from ..core import Metadata, Media, ArchivingContext from ..core import Metadata, Media, ArchivingContext
from . import Formatter from . import Formatter
from ..enrichers import HashEnricher from ..enrichers import HashEnricher
from ..utils.misc import random_str
@dataclass @dataclass
@ -44,7 +45,7 @@ class HtmlFormatter(Formatter):
metadata=item.metadata, metadata=item.metadata,
version=__version__ version=__version__
) )
html_path = os.path.join(ArchivingContext.get_tmp_dir(), f"formatted{str(uuid.uuid4())}.html") html_path = os.path.join(ArchivingContext.get_tmp_dir(), f"formatted{random_str(24)}.html")
with open(html_path, mode="w", encoding="utf-8") as outf: with open(html_path, mode="w", encoding="utf-8") as outf:
outf.write(content) outf.write(content)
final_media = Media(filename=html_path, _mimetype="text/html") final_media = Media(filename=html_path, _mimetype="text/html")

69
src/auto_archiver/storages/s3.py
Wyświetl plik

@ -1,14 +1,14 @@
from typing import IO, Any from typing import IO
import boto3, uuid, os, mimetypes import boto3, os
from botocore.errorfactory import ClientError
from ..core import Metadata from ..utils.misc import random_str
from ..core import Media from ..core import Media
from ..storages import Storage from ..storages import Storage
from ..enrichers import HashEnricher
from loguru import logger from loguru import logger
from slugify import slugify
NO_DUPLICATES_FOLDER = "no-dups/"
class S3Storage(Storage): class S3Storage(Storage):
name = "s3_storage" name = "s3_storage"
@ -21,6 +21,9 @@ class S3Storage(Storage):
aws_access_key_id=self.key, aws_access_key_id=self.key,
aws_secret_access_key=self.secret aws_secret_access_key=self.secret
) )
self.random_no_duplicate = bool(self.random_no_duplicate)
if self.random_no_duplicate:
logger.warning("random_no_duplicate is set to True, this will override `path_generator`, `filename_generator` and `folder`.")
@staticmethod @staticmethod
def configs() -> dict: def configs() -> dict:
@ -31,7 +34,7 @@ class S3Storage(Storage):
"region": {"default": None, "help": "S3 region name"}, "region": {"default": None, "help": "S3 region name"},
"key": {"default": None, "help": "S3 API key"}, "key": {"default": None, "help": "S3 API key"},
"secret": {"default": None, "help": "S3 API secret"}, "secret": {"default": None, "help": "S3 API secret"},
# TODO: how to have sth like a custom folder? has to come from the feeders "random_no_duplicate": {"default": False, "help": f"if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `{NO_DUPLICATES_FOLDER}`"},
"endpoint_url": { "endpoint_url": {
"default": 'https://{region}.digitaloceanspaces.com', "default": 'https://{region}.digitaloceanspaces.com',
"help": "S3 bucket endpoint, {region} are inserted at runtime" "help": "S3 bucket endpoint, {region} are inserted at runtime"
@ -47,6 +50,22 @@ class S3Storage(Storage):
return self.cdn_url.format(bucket=self.bucket, region=self.region, key=media.key) return self.cdn_url.format(bucket=self.bucket, region=self.region, key=media.key)
def uploadf(self, file: IO[bytes], media: Media, **kwargs: dict) -> None: def uploadf(self, file: IO[bytes], media: Media, **kwargs: dict) -> None:
if not self.is_upload_needed(media): return True
if self.random_no_duplicate:
# checks if a folder with the hash already exists, if so it skips the upload
he = HashEnricher({"hash_enricher": {"algorithm": "SHA-256", "chunksize": 1.6e7}})
hd = he.calculate_hash(media.filename)
path = os.path.join(NO_DUPLICATES_FOLDER, hd[:24])
if existing_key:=self.file_in_folder(path):
media.key = existing_key
logger.debug(f"skipping upload of {media.filename} because it already exists in {media.key}")
return True
_, ext = os.path.splitext(media.key)
media.key = os.path.join(path, f"{random_str(24)}{ext}")
extra_args = kwargs.get("extra_args", {}) extra_args = kwargs.get("extra_args", {})
if not self.private and 'ACL' not in extra_args: if not self.private and 'ACL' not in extra_args:
extra_args['ACL'] = 'public-read' extra_args['ACL'] = 'public-read'
@ -61,13 +80,29 @@ class S3Storage(Storage):
self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args) self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args)
return True return True
# def exists(self, key: str) -> bool: def is_upload_needed(self, media: Media) -> bool:
# """ if self.random_no_duplicate:
# Tests if a given file with key=key exists in the bucket # checks if a folder with the hash already exists, if so it skips the upload
# """ he = HashEnricher({"hash_enricher": {"algorithm": "SHA-256", "chunksize": 1.6e7}})
# try: hd = he.calculate_hash(media.filename)
# self.s3.head_object(Bucket=self.bucket, Key=key) path = os.path.join(NO_DUPLICATES_FOLDER, hd[:24])
# return True
# except ClientError as e: if existing_key:=self.file_in_folder(path):
# logger.warning(f"got a ClientError when checking if {key=} exists in bucket={self.bucket}: {e}") media.key = existing_key
# return False logger.debug(f"skipping upload of {media.filename} because it already exists in {media.key}")
return False
_, ext = os.path.splitext(media.key)
media.key = os.path.join(path, f"{random_str(24)}{ext}")
return True
def file_in_folder(self, path:str) -> str:
# checks if path exists and is not an empty folder
if not path.endswith('/'):
path = path + '/'
resp = self.s3.list_objects(Bucket=self.bucket, Prefix=path, Delimiter='/', MaxKeys=1)
if 'Contents' in resp:
return resp['Contents'][0]['Key']
return False

8
src/auto_archiver/storages/storage.py
Wyświetl plik

@ -2,11 +2,13 @@ from __future__ import annotations
from abc import abstractmethod from abc import abstractmethod
from dataclasses import dataclass from dataclasses import dataclass
from typing import IO from typing import IO
import os
from ..utils.misc import random_str
from ..core import Media, Step, ArchivingContext from ..core import Media, Step, ArchivingContext
from ..enrichers import HashEnricher from ..enrichers import HashEnricher
from loguru import logger from loguru import logger
import os, uuid
from slugify import slugify from slugify import slugify
@ -72,10 +74,10 @@ class Storage(Step):
filename = slugify(filename) # in case it comes with os.sep filename = slugify(filename) # in case it comes with os.sep
elif self.path_generator == "url": path = slugify(url) elif self.path_generator == "url": path = slugify(url)
elif self.path_generator == "random": elif self.path_generator == "random":
path = ArchivingContext.get("random_path", str(uuid.uuid4())[:16], True) path = ArchivingContext.get("random_path", random_str(24), True)
# filename_generator logic # filename_generator logic
if self.filename_generator == "random": filename = str(uuid.uuid4())[:16] if self.filename_generator == "random": filename = random_str(24)
elif self.filename_generator == "static": elif self.filename_generator == "static":
he = HashEnricher({"hash_enricher": {"algorithm": ArchivingContext.get("hash_enricher.algorithm"), "chunksize": 1.6e7}}) he = HashEnricher({"hash_enricher": {"algorithm": ArchivingContext.get("hash_enricher.algorithm"), "chunksize": 1.6e7}})
hd = he.calculate_hash(media.filename) hd = he.calculate_hash(media.filename)

5
src/auto_archiver/utils/misc.py
Wyświetl plik

@ -1,5 +1,6 @@
import os, json, requests import os, json, requests
import uuid
from datetime import datetime from datetime import datetime
from loguru import logger from loguru import logger
@ -49,3 +50,7 @@ def update_nested_dict(dictionary, update_dict):
update_nested_dict(dictionary[key], value) update_nested_dict(dictionary[key], value)
else: else:
dictionary[key] = value dictionary[key] = value
def random_str(length: int = 32) -> str:
assert length <= 32, "length must be less than 32 as UUID4 is used"
return str(uuid.uuid4()).replace("-", "")[:length]