From 7a21ae96afb184b9e06bb19f5e5bf94af4a3396f Mon Sep 17 00:00:00 2001 From: Miguel Sozinho Ramalho <19508417+msramalho@users.noreply.github.com> Date: Tue, 20 Feb 2024 18:05:29 +0000 Subject: [PATCH] V0.9.0 - closes several open issues: new enrichers and bug fixes (#133) * clean orchestrator code, add archiver cleanup logic * improves documentation for database.py * telethon archivers isolate sessions into copied files * closes #127 * closes #125 * closes #84 * meta enricher applies to all media * closes #61 adds subtitles and comments * minor update * minor fixes to yt-dlp subtitles and comments * closes #17 but logic is imperfect. * closes #85 ssl enhancer * minimifies html, JS refactor for preview of certificates * closes #91 adds freetsa timestamp authority * version bump * simplify download_url method * skip ssl if nothing archived * html preview improvements * adds retrying lib * manual download archiver improvements * meta only runs when relevant data available * new metadata convenience method * html template improvements * removes debug message * does not close #91 yet, will need a few more certificate chaing logging * adds verbosity config * new instagram api archiver * adds proxy support we * adds proxy/end support and bug fix for yt-dlp * proxy support for webdriver * adds socks proxy to wacz_enricher * refactor recursivity in inner media and display * infinite recursive display * foolproofing timestamping authortities * version to 0.9.0 * minor fixes from code-review --- .github/workflows/docker-publish.yaml | 3 - .github/workflows/python-publish.yaml | 3 - Pipfile | 5 + Pipfile.lock | 1399 +++++++++-------- README.md | 8 +- scripts/release.sh | 19 - src/auto_archiver/archivers/__init__.py | 3 +- src/auto_archiver/archivers/archiver.py | 15 +- .../archivers/instagram_api_archiver.py | 272 ++++ .../archivers/instagram_tbot_archiver.py | 26 +- .../archivers/telegram_archiver.py | 4 +- .../archivers/telethon_archiver.py | 25 +- .../archivers/twitter_api_archiver.py | 2 +- .../archivers/twitter_archiver.py | 4 +- .../archivers/youtubedl_archiver.py | 59 +- src/auto_archiver/core/context.py | 3 - src/auto_archiver/core/media.py | 8 +- src/auto_archiver/core/metadata.py | 9 +- src/auto_archiver/core/orchestrator.py | 66 +- src/auto_archiver/databases/database.py | 2 +- src/auto_archiver/enrichers/__init__.py | 4 +- src/auto_archiver/enrichers/meta_enricher.py | 20 +- .../enrichers/screenshot_enricher.py | 5 +- src/auto_archiver/enrichers/ssl_enricher.py | 36 + .../enrichers/thumbnail_enricher.py | 58 +- .../enrichers/timestamping_enricher.py | 136 ++ src/auto_archiver/enrichers/wacz_enricher.py | 13 +- .../enrichers/wayback_enricher.py | 21 +- .../enrichers/whisper_enricher.py | 2 +- .../formatters/html_formatter.py | 9 + .../formatters/templates/html_template.html | 249 +-- .../formatters/templates/macros.html | 79 +- src/auto_archiver/utils/webdriver.py | 5 +- src/auto_archiver/version.py | 4 +- 34 files changed, 1696 insertions(+), 880 deletions(-) delete mode 100755 scripts/release.sh create mode 100644 src/auto_archiver/archivers/instagram_api_archiver.py create mode 100644 src/auto_archiver/enrichers/ssl_enricher.py create mode 100644 src/auto_archiver/enrichers/timestamping_enricher.py diff --git a/.github/workflows/docker-publish.yaml b/.github/workflows/docker-publish.yaml index 44ca808..379aaaa 100644 --- a/.github/workflows/docker-publish.yaml +++ b/.github/workflows/docker-publish.yaml @@ -8,9 +8,6 @@ name: Docker on: release: types: [published] - push: - # branches: [ "main" ] - tags: [ "v*.*.*" ] env: # Use docker.io for Docker Hub if empty diff --git a/.github/workflows/python-publish.yaml b/.github/workflows/python-publish.yaml index cbd9c5a..73e19c6 100644 --- a/.github/workflows/python-publish.yaml +++ b/.github/workflows/python-publish.yaml @@ -11,9 +11,6 @@ name: Pypi on: release: types: [published] - push: - # branches: [ "main" ] - tags: [ "v*.*.*" ] permissions: contents: read diff --git a/Pipfile b/Pipfile index 1fb0904..e94fb28 100644 --- a/Pipfile +++ b/Pipfile @@ -36,6 +36,11 @@ requests = {extras = ["socks"], version = "*"} numpy = "*" warcio = "*" jsonlines = "*" +pysubs2 = "*" +minify-html = "*" +retrying = "*" +tsp-client = "*" +certvalidator = "*" [dev-packages] autopep8 = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 83f9342..f6797c5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "48785b4cb4cf34066866e48f54493b9fb5583cbb42055c402d968d5e80c0c610" + "sha256": "7fb3af40d6914f80cf7c0d3750985adbdbddf60501abba396436541fd63f121c" }, "pipfile-spec": 6, "requires": { @@ -18,85 +18,85 @@ "default": { "aiohttp": { "hashes": [ - "sha256:02ab6006ec3c3463b528374c4cdce86434e7b89ad355e7bf29e2f16b46c7dd6f", - "sha256:04fa38875e53eb7e354ece1607b1d2fdee2d175ea4e4d745f6ec9f751fe20c7c", - "sha256:0b0a6a36ed7e164c6df1e18ee47afbd1990ce47cb428739d6c99aaabfaf1b3af", - "sha256:0d406b01a9f5a7e232d1b0d161b40c05275ffbcbd772dc18c1d5a570961a1ca4", - "sha256:0e49b08eafa4f5707ecfb321ab9592717a319e37938e301d462f79b4e860c32a", - "sha256:0e7ba7ff228c0d9a2cd66194e90f2bca6e0abca810b786901a569c0de082f489", - "sha256:11cb254e397a82efb1805d12561e80124928e04e9c4483587ce7390b3866d213", - "sha256:11ff168d752cb41e8492817e10fb4f85828f6a0142b9726a30c27c35a1835f01", - "sha256:176df045597e674fa950bf5ae536be85699e04cea68fa3a616cf75e413737eb5", - "sha256:219a16763dc0294842188ac8a12262b5671817042b35d45e44fd0a697d8c8361", - "sha256:22698f01ff5653fe66d16ffb7658f582a0ac084d7da1323e39fd9eab326a1f26", - "sha256:237533179d9747080bcaad4d02083ce295c0d2eab3e9e8ce103411a4312991a0", - "sha256:289ba9ae8e88d0ba16062ecf02dd730b34186ea3b1e7489046fc338bdc3361c4", - "sha256:2c59e0076ea31c08553e868cec02d22191c086f00b44610f8ab7363a11a5d9d8", - "sha256:2c9376e2b09895c8ca8b95362283365eb5c03bdc8428ade80a864160605715f1", - "sha256:3135713c5562731ee18f58d3ad1bf41e1d8883eb68b363f2ffde5b2ea4b84cc7", - "sha256:3b9c7426923bb7bd66d409da46c41e3fb40f5caf679da624439b9eba92043fa6", - "sha256:3c0266cd6f005e99f3f51e583012de2778e65af6b73860038b968a0a8888487a", - "sha256:41473de252e1797c2d2293804e389a6d6986ef37cbb4a25208de537ae32141dd", - "sha256:4831df72b053b1eed31eb00a2e1aff6896fb4485301d4ccb208cac264b648db4", - "sha256:49f0c1b3c2842556e5de35f122fc0f0b721334ceb6e78c3719693364d4af8499", - "sha256:4b4c452d0190c5a820d3f5c0f3cd8a28ace48c54053e24da9d6041bf81113183", - "sha256:4ee8caa925aebc1e64e98432d78ea8de67b2272252b0a931d2ac3bd876ad5544", - "sha256:500f1c59906cd142d452074f3811614be04819a38ae2b3239a48b82649c08821", - "sha256:5216b6082c624b55cfe79af5d538e499cd5f5b976820eac31951fb4325974501", - "sha256:54311eb54f3a0c45efb9ed0d0a8f43d1bc6060d773f6973efd90037a51cd0a3f", - "sha256:54631fb69a6e44b2ba522f7c22a6fb2667a02fd97d636048478db2fd8c4e98fe", - "sha256:565760d6812b8d78d416c3c7cfdf5362fbe0d0d25b82fed75d0d29e18d7fc30f", - "sha256:598db66eaf2e04aa0c8900a63b0101fdc5e6b8a7ddd805c56d86efb54eb66672", - "sha256:5c4fa235d534b3547184831c624c0b7c1e262cd1de847d95085ec94c16fddcd5", - "sha256:69985d50a2b6f709412d944ffb2e97d0be154ea90600b7a921f95a87d6f108a2", - "sha256:69da0f3ed3496808e8cbc5123a866c41c12c15baaaead96d256477edf168eb57", - "sha256:6c93b7c2e52061f0925c3382d5cb8980e40f91c989563d3d32ca280069fd6a87", - "sha256:70907533db712f7aa791effb38efa96f044ce3d4e850e2d7691abd759f4f0ae0", - "sha256:81b77f868814346662c96ab36b875d7814ebf82340d3284a31681085c051320f", - "sha256:82eefaf1a996060602f3cc1112d93ba8b201dbf5d8fd9611227de2003dddb3b7", - "sha256:85c3e3c9cb1d480e0b9a64c658cd66b3cfb8e721636ab8b0e746e2d79a7a9eed", - "sha256:8a22a34bc594d9d24621091d1b91511001a7eea91d6652ea495ce06e27381f70", - "sha256:8cef8710fb849d97c533f259103f09bac167a008d7131d7b2b0e3a33269185c0", - "sha256:8d44e7bf06b0c0a70a20f9100af9fcfd7f6d9d3913e37754c12d424179b4e48f", - "sha256:8d7f98fde213f74561be1d6d3fa353656197f75d4edfbb3d94c9eb9b0fc47f5d", - "sha256:8d8e4450e7fe24d86e86b23cc209e0023177b6d59502e33807b732d2deb6975f", - "sha256:8fc49a87ac269d4529da45871e2ffb6874e87779c3d0e2ccd813c0899221239d", - "sha256:90ec72d231169b4b8d6085be13023ece8fa9b1bb495e4398d847e25218e0f431", - "sha256:91c742ca59045dce7ba76cab6e223e41d2c70d79e82c284a96411f8645e2afff", - "sha256:9b05d33ff8e6b269e30a7957bd3244ffbce2a7a35a81b81c382629b80af1a8bf", - "sha256:9b05d5cbe9dafcdc733262c3a99ccf63d2f7ce02543620d2bd8db4d4f7a22f83", - "sha256:9c5857612c9813796960c00767645cb5da815af16dafb32d70c72a8390bbf690", - "sha256:a34086c5cc285be878622e0a6ab897a986a6e8bf5b67ecb377015f06ed316587", - "sha256:ab221850108a4a063c5b8a70f00dd7a1975e5a1713f87f4ab26a46e5feac5a0e", - "sha256:b796b44111f0cab6bbf66214186e44734b5baab949cb5fb56154142a92989aeb", - "sha256:b8c3a67eb87394386847d188996920f33b01b32155f0a94f36ca0e0c635bf3e3", - "sha256:bcb6532b9814ea7c5a6a3299747c49de30e84472fa72821b07f5a9818bce0f66", - "sha256:bcc0ea8d5b74a41b621ad4a13d96c36079c81628ccc0b30cfb1603e3dfa3a014", - "sha256:bea94403a21eb94c93386d559bce297381609153e418a3ffc7d6bf772f59cc35", - "sha256:bff7e2811814fa2271be95ab6e84c9436d027a0e59665de60edf44e529a42c1f", - "sha256:c72444d17777865734aa1a4d167794c34b63e5883abb90356a0364a28904e6c0", - "sha256:c7b5d5d64e2a14e35a9240b33b89389e0035e6de8dbb7ffa50d10d8b65c57449", - "sha256:c7e939f1ae428a86e4abbb9a7c4732bf4706048818dfd979e5e2839ce0159f23", - "sha256:c88a15f272a0ad3d7773cf3a37cc7b7d077cbfc8e331675cf1346e849d97a4e5", - "sha256:c9110c06eaaac7e1f5562caf481f18ccf8f6fdf4c3323feab28a93d34cc646bd", - "sha256:ca7ca5abfbfe8d39e653870fbe8d7710be7a857f8a8386fc9de1aae2e02ce7e4", - "sha256:cae4c0c2ca800c793cae07ef3d40794625471040a87e1ba392039639ad61ab5b", - "sha256:cdefe289681507187e375a5064c7599f52c40343a8701761c802c1853a504558", - "sha256:cf2a0ac0615842b849f40c4d7f304986a242f1e68286dbf3bd7a835e4f83acfd", - "sha256:cfeadf42840c1e870dc2042a232a8748e75a36b52d78968cda6736de55582766", - "sha256:d737e69d193dac7296365a6dcb73bbbf53bb760ab25a3727716bbd42022e8d7a", - "sha256:d7481f581251bb5558ba9f635db70908819caa221fc79ee52a7f58392778c636", - "sha256:df9cf74b9bc03d586fc53ba470828d7b77ce51b0582d1d0b5b2fb673c0baa32d", - "sha256:e1f80197f8b0b846a8d5cf7b7ec6084493950d0882cc5537fb7b96a69e3c8590", - "sha256:ecca113f19d5e74048c001934045a2b9368d77b0b17691d905af18bd1c21275e", - "sha256:ee2527134f95e106cc1653e9ac78846f3a2ec1004cf20ef4e02038035a74544d", - "sha256:f27fdaadce22f2ef950fc10dcdf8048407c3b42b73779e48a4e76b3c35bca26c", - "sha256:f694dc8a6a3112059258a725a4ebe9acac5fe62f11c77ac4dcf896edfa78ca28", - "sha256:f800164276eec54e0af5c99feb9494c295118fc10a11b997bbb1348ba1a52065", - "sha256:ffcd828e37dc219a72c9012ec44ad2e7e3066bec6ff3aaa19e7d435dbf4032ca" + "sha256:017a21b0df49039c8f46ca0971b3a7fdc1f56741ab1240cb90ca408049766168", + "sha256:039df344b45ae0b34ac885ab5b53940b174530d4dd8a14ed8b0e2155b9dddccb", + "sha256:055ce4f74b82551678291473f66dc9fb9048a50d8324278751926ff0ae7715e5", + "sha256:06a9b2c8837d9a94fae16c6223acc14b4dfdff216ab9b7202e07a9a09541168f", + "sha256:07b837ef0d2f252f96009e9b8435ec1fef68ef8b1461933253d318748ec1acdc", + "sha256:0ed621426d961df79aa3b963ac7af0d40392956ffa9be022024cd16297b30c8c", + "sha256:0fa43c32d1643f518491d9d3a730f85f5bbaedcbd7fbcae27435bb8b7a061b29", + "sha256:1f5a71d25cd8106eab05f8704cd9167b6e5187bcdf8f090a66c6d88b634802b4", + "sha256:1f5cd333fcf7590a18334c90f8c9147c837a6ec8a178e88d90a9b96ea03194cc", + "sha256:27468897f628c627230dba07ec65dc8d0db566923c48f29e084ce382119802bc", + "sha256:298abd678033b8571995650ccee753d9458dfa0377be4dba91e4491da3f2be63", + "sha256:2c895a656dd7e061b2fd6bb77d971cc38f2afc277229ce7dd3552de8313a483e", + "sha256:361a1026c9dd4aba0109e4040e2aecf9884f5cfe1b1b1bd3d09419c205e2e53d", + "sha256:363afe77cfcbe3a36353d8ea133e904b108feea505aa4792dad6585a8192c55a", + "sha256:38a19bc3b686ad55804ae931012f78f7a534cce165d089a2059f658f6c91fa60", + "sha256:38f307b41e0bea3294a9a2a87833191e4bcf89bb0365e83a8be3a58b31fb7f38", + "sha256:3e59c23c52765951b69ec45ddbbc9403a8761ee6f57253250c6e1536cacc758b", + "sha256:4b4af9f25b49a7be47c0972139e59ec0e8285c371049df1a63b6ca81fdd216a2", + "sha256:504b6981675ace64c28bf4a05a508af5cde526e36492c98916127f5a02354d53", + "sha256:50fca156d718f8ced687a373f9e140c1bb765ca16e3d6f4fe116e3df7c05b2c5", + "sha256:522a11c934ea660ff8953eda090dcd2154d367dec1ae3c540aff9f8a5c109ab4", + "sha256:52df73f14ed99cee84865b95a3d9e044f226320a87af208f068ecc33e0c35b96", + "sha256:595f105710293e76b9dc09f52e0dd896bd064a79346234b521f6b968ffdd8e58", + "sha256:59c26c95975f26e662ca78fdf543d4eeaef70e533a672b4113dd888bd2423caa", + "sha256:5bce0dc147ca85caa5d33debc4f4d65e8e8b5c97c7f9f660f215fa74fc49a321", + "sha256:5eafe2c065df5401ba06821b9a054d9cb2848867f3c59801b5d07a0be3a380ae", + "sha256:5ed3e046ea7b14938112ccd53d91c1539af3e6679b222f9469981e3dac7ba1ce", + "sha256:5fe9ce6c09668063b8447f85d43b8d1c4e5d3d7e92c63173e6180b2ac5d46dd8", + "sha256:648056db9a9fa565d3fa851880f99f45e3f9a771dd3ff3bb0c048ea83fb28194", + "sha256:69361bfdca5468c0488d7017b9b1e5ce769d40b46a9f4a2eed26b78619e9396c", + "sha256:6b0e029353361f1746bac2e4cc19b32f972ec03f0f943b390c4ab3371840aabf", + "sha256:6b88f9386ff1ad91ace19d2a1c0225896e28815ee09fc6a8932fded8cda97c3d", + "sha256:770d015888c2a598b377bd2f663adfd947d78c0124cfe7b959e1ef39f5b13869", + "sha256:7943c414d3a8d9235f5f15c22ace69787c140c80b718dcd57caaade95f7cd93b", + "sha256:7cf5c9458e1e90e3c390c2639f1017a0379a99a94fdfad3a1fd966a2874bba52", + "sha256:7f46acd6a194287b7e41e87957bfe2ad1ad88318d447caf5b090012f2c5bb528", + "sha256:82e6aa28dd46374f72093eda8bcd142f7771ee1eb9d1e223ff0fa7177a96b4a5", + "sha256:835a55b7ca49468aaaac0b217092dfdff370e6c215c9224c52f30daaa735c1c1", + "sha256:84871a243359bb42c12728f04d181a389718710129b36b6aad0fc4655a7647d4", + "sha256:8aacb477dc26797ee089721536a292a664846489c49d3ef9725f992449eda5a8", + "sha256:8e2c45c208c62e955e8256949eb225bd8b66a4c9b6865729a786f2aa79b72e9d", + "sha256:90842933e5d1ff760fae6caca4b2b3edba53ba8f4b71e95dacf2818a2aca06f7", + "sha256:938a9653e1e0c592053f815f7028e41a3062e902095e5a7dc84617c87267ebd5", + "sha256:939677b61f9d72a4fa2a042a5eee2a99a24001a67c13da113b2e30396567db54", + "sha256:9d3c9b50f19704552f23b4eaea1fc082fdd82c63429a6506446cbd8737823da3", + "sha256:a6fe5571784af92b6bc2fda8d1925cccdf24642d49546d3144948a6a1ed58ca5", + "sha256:a78ed8a53a1221393d9637c01870248a6f4ea5b214a59a92a36f18151739452c", + "sha256:ab40e6251c3873d86ea9b30a1ac6d7478c09277b32e14745d0d3c6e76e3c7e29", + "sha256:abf151955990d23f84205286938796c55ff11bbfb4ccfada8c9c83ae6b3c89a3", + "sha256:acef0899fea7492145d2bbaaaec7b345c87753168589cc7faf0afec9afe9b747", + "sha256:b40670ec7e2156d8e57f70aec34a7216407848dfe6c693ef131ddf6e76feb672", + "sha256:b791a3143681a520c0a17e26ae7465f1b6f99461a28019d1a2f425236e6eedb5", + "sha256:b955ed993491f1a5da7f92e98d5dad3c1e14dc175f74517c4e610b1f2456fb11", + "sha256:ba39e9c8627edc56544c8628cc180d88605df3892beeb2b94c9bc857774848ca", + "sha256:bca77a198bb6e69795ef2f09a5f4c12758487f83f33d63acde5f0d4919815768", + "sha256:c3452ea726c76e92f3b9fae4b34a151981a9ec0a4847a627c43d71a15ac32aa6", + "sha256:c46956ed82961e31557b6857a5ca153c67e5476972e5f7190015018760938da2", + "sha256:c7c8b816c2b5af5c8a436df44ca08258fc1a13b449393a91484225fcb7545533", + "sha256:cd73265a9e5ea618014802ab01babf1940cecb90c9762d8b9e7d2cc1e1969ec6", + "sha256:dad46e6f620574b3b4801c68255492e0159d1712271cc99d8bdf35f2043ec266", + "sha256:dc9b311743a78043b26ffaeeb9715dc360335e5517832f5a8e339f8a43581e4d", + "sha256:df822ee7feaaeffb99c1a9e5e608800bd8eda6e5f18f5cfb0dc7eeb2eaa6bbec", + "sha256:e083c285857b78ee21a96ba1eb1b5339733c3563f72980728ca2b08b53826ca5", + "sha256:e5e46b578c0e9db71d04c4b506a2121c0cb371dd89af17a0586ff6769d4c58c1", + "sha256:e99abf0bba688259a496f966211c49a514e65afa9b3073a1fcee08856e04425b", + "sha256:ee43080e75fc92bf36219926c8e6de497f9b247301bbf88c5c7593d931426679", + "sha256:f033d80bc6283092613882dfe40419c6a6a1527e04fc69350e87a9df02bbc283", + "sha256:f1088fa100bf46e7b398ffd9904f4808a0612e1d966b4aa43baa535d1b6341eb", + "sha256:f56455b0c2c7cc3b0c584815264461d07b177f903a04481dfc33e08a89f0c26b", + "sha256:f59dfe57bb1ec82ac0698ebfcdb7bcd0e99c255bd637ff613760d5f33e7c81b3", + "sha256:f7217af2e14da0856e082e96ff637f14ae45c10a5714b63c77f26d8884cf1051", + "sha256:f734e38fd8666f53da904c52a23ce517f1b07722118d750405af7e4123933511", + "sha256:f95511dd5d0e05fd9728bac4096319f80615aaef4acbecb35a990afebe953b0e", + "sha256:fdd215b7b7fd4a53994f238d0f46b7ba4ac4c0adb12452beee724ddd0743ae5d", + "sha256:feeb18a801aacb098220e2c3eea59a512362eb408d4afd0c242044c33ad6d542", + "sha256:ff30218887e62209942f91ac1be902cc80cddb86bf00fbc6783b7a43b2bea26f" ], "markers": "python_version >= '3.8'", - "version": "==3.9.1" + "version": "==3.9.3" }, "aiosignal": { "hashes": [ @@ -108,11 +108,11 @@ }, "anyio": { "hashes": [ - "sha256:745843b39e829e108e518c489b31dc757de7d2131d53fac32bd8df268227bfee", - "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f" + "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8", + "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6" ], "markers": "python_version >= '3.8'", - "version": "==4.2.0" + "version": "==4.3.0" }, "argparse": { "hashes": [ @@ -122,6 +122,13 @@ "index": "pypi", "version": "==1.4.0" }, + "asn1crypto": { + "hashes": [ + "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", + "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67" + ], + "version": "==1.5.1" + }, "async-timeout": { "hashes": [ "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f", @@ -132,11 +139,11 @@ }, "attrs": { "hashes": [ - "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04", - "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015" + "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30", + "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1" ], "markers": "python_version >= '3.7'", - "version": "==23.1.0" + "version": "==23.2.0" }, "authlib": { "hashes": [ @@ -148,12 +155,12 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da", - "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a" + "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051", + "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed" ], "index": "pypi", "markers": "python_full_version >= '3.6.0'", - "version": "==4.12.2" + "version": "==4.12.3" }, "blinker": { "hashes": [ @@ -165,20 +172,20 @@ }, "boto3": { "hashes": [ - "sha256:1e836fe33da2684db29317911d9958389094ca5098cc253dbaed8e4aa146b153", - "sha256:a866277fc38b121ac5dab0eec38b6ae6e3a59bbf6f67ed9a9822332d9e5e785f" + "sha256:46432fd506708fec6caec4392d758c6f5b79a376dee67d3284fe8b6bfbafeaf4", + "sha256:5c96bed1269f77788780aa2005811dc3a37d4122f08b8e54063a1f4c1b9314a1" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.34.4" + "version": "==1.34.45" }, "botocore": { "hashes": [ - "sha256:2026d89a46dfcb96d439db17a277de11b808428cba881deb50a5960b134e3a84", - "sha256:5dcd63329cb3e65c533a72a68c99b7d07c99a29936ea07d0998120172c10b4f5" + "sha256:bf4fe24dd00a6262a27573dea1690ea68eb20f939e7086effadf19aa1acb44d1", + "sha256:e17874ac708fef295d2ea16bb2570ea0512c920de9f25f796de0d8c778f06a02" ], "markers": "python_version >= '3.8'", - "version": "==1.34.4" + "version": "==1.34.45" }, "brotli": { "hashes": [ @@ -271,10 +278,11 @@ }, "bs4": { "hashes": [ - "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a" + "sha256:a48685c58f50fe127722417bae83fe6badf500d54b55f7e39ffe43b798653925", + "sha256:abf8742c0805ef7f662dce4b51cca104cffe52b835238afc169142ab9b3fbccc" ], "index": "pypi", - "version": "==0.0.1" + "version": "==0.0.2" }, "cachetools": { "hashes": [ @@ -286,11 +294,19 @@ }, "certifi": { "hashes": [ - "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1", - "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474" + "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f", + "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1" ], "markers": "python_version >= '3.6'", - "version": "==2023.11.17" + "version": "==2024.2.2" + }, + "certvalidator": { + "hashes": [ + "sha256:77520b269f516d4fb0902998d5bd0eb3727fe153b659aa1cb828dcf12ea6b8de", + "sha256:922d141c94393ab285ca34338e18dd4093e3ae330b1f278e96c837cb62cffaad" + ], + "index": "pypi", + "version": "==0.11.1" }, "cffi": { "hashes": [ @@ -347,7 +363,7 @@ "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956", "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357" ], - "markers": "python_version >= '3.8'", + "markers": "platform_python_implementation != 'PyPy'", "version": "==1.16.0" }, "charset-normalizer": { @@ -443,7 +459,7 @@ "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.7.0'", "version": "==3.3.2" }, "click": { @@ -463,33 +479,42 @@ }, "cryptography": { "hashes": [ - "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960", - "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a", - "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc", - "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a", - "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf", - "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1", - "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39", - "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406", - "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a", - "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a", - "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c", - "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be", - "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15", - "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2", - "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d", - "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157", - "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003", - "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248", - "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a", - "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec", - "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309", - "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7", - "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d" + "sha256:04859aa7f12c2b5f7e22d25198ddd537391f1695df7057c8700f71f26f47a129", + "sha256:069d2ce9be5526a44093a0991c450fe9906cdf069e0e7cd67d9dee49a62b9ebe", + "sha256:0d3ec384058b642f7fb7e7bff9664030011ed1af8f852540c76a1317a9dd0d20", + "sha256:0fab2a5c479b360e5e0ea9f654bcebb535e3aa1e493a715b13244f4e07ea8eec", + "sha256:0fea01527d4fb22ffe38cd98951c9044400f6eff4788cf52ae116e27d30a1ba3", + "sha256:1b797099d221df7cce5ff2a1d272761d1554ddf9a987d3e11f6459b38cd300fd", + "sha256:1e935c2900fb53d31f491c0de04f41110351377be19d83d908c1fd502ae8daa5", + "sha256:20100c22b298c9eaebe4f0b9032ea97186ac2555f426c3e70670f2517989543b", + "sha256:20180da1b508f4aefc101cebc14c57043a02b355d1a652b6e8e537967f1e1b46", + "sha256:25b09b73db78facdfd7dd0fa77a3f19e94896197c86e9f6dc16bce7b37a96504", + "sha256:2619487f37da18d6826e27854a7f9d4d013c51eafb066c80d09c63cf24505306", + "sha256:2eb6368d5327d6455f20327fb6159b97538820355ec00f8cc9464d617caecead", + "sha256:35772a6cffd1f59b85cb670f12faba05513446f80352fe811689b4e439b5d89e", + "sha256:39d5c93e95bcbc4c06313fc6a500cee414ee39b616b55320c1904760ad686938", + "sha256:3d96ea47ce6d0055d5b97e761d37b4e84195485cb5a38401be341fabf23bc32a", + "sha256:4dcab7c25e48fc09a73c3e463d09ac902a932a0f8d0c568238b3696d06bf377b", + "sha256:5fbf0f3f0fac7c089308bd771d2c6c7b7d53ae909dce1db52d8e921f6c19bb3a", + "sha256:6c25e1e9c2ce682d01fc5e2dde6598f7313027343bd14f4049b82ad0402e52cd", + "sha256:762f3771ae40e111d78d77cbe9c1035e886ac04a234d3ee0856bf4ecb3749d54", + "sha256:90147dad8c22d64b2ff7331f8d4cddfdc3ee93e4879796f837bdbb2a0b141e0c", + "sha256:935cca25d35dda9e7bd46a24831dfd255307c55a07ff38fd1a92119cffc34857", + "sha256:93fbee08c48e63d5d1b39ab56fd3fdd02e6c2431c3da0f4edaf54954744c718f", + "sha256:9541c69c62d7446539f2c1c06d7046aef822940d248fa4b8962ff0302862cc1f", + "sha256:c23f03cfd7d9826cdcbad7850de67e18b4654179e01fe9bc623d37c2638eb4ef", + "sha256:c3d1f5a1d403a8e640fa0887e9f7087331abb3f33b0f2207d2cc7f213e4a864c", + "sha256:d1998e545081da0ab276bcb4b33cce85f775adb86a516e8f55b3dac87f469548", + "sha256:d5cf11bc7f0b71fb71af26af396c83dfd3f6eed56d4b6ef95d57867bf1e4ba65", + "sha256:db0480ffbfb1193ac4e1e88239f31314fe4c6cdcf9c0b8712b55414afbf80db4", + "sha256:de4ae486041878dc46e571a4c70ba337ed5233a1344c14a0790c4c4be4bbb8b4", + "sha256:de5086cd475d67113ccb6f9fae6d8fe3ac54a4f9238fd08bfdb07b03d791ff0a", + "sha256:df34312149b495d9d03492ce97471234fd9037aa5ba217c2a6ea890e9166f151", + "sha256:ead69ba488f806fe1b1b4050febafdbf206b81fa476126f3e16110c818bac396" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==41.0.7" + "version": "==42.0.3" }, "dataclasses-json": { "hashes": [ @@ -535,11 +560,11 @@ }, "flask": { "hashes": [ - "sha256:21128f47e4e3b9d597a3e8521a329bf56909b690fcc3fa3e477725aa81367638", - "sha256:cfadcdb638b609361d29ec22360d6070a77d7463dcb3ab08d2c2f2f168845f58" + "sha256:3232e0e9c850d781933cf0207523d1ece087eb8d87b23777ae38456e2fbe7c6e", + "sha256:822c03f4b799204250a7ee84b1eddc40665395333973dfb9deebfe425fefcb7d" ], "markers": "python_version >= '3.8'", - "version": "==3.0.0" + "version": "==3.0.2" }, "frozenlist": { "hashes": [ @@ -633,28 +658,28 @@ }, "google-api-core": { "hashes": [ - "sha256:2aa56d2be495551e66bbff7f729b790546f87d5c90e74781aa77233bcb395a8a", - "sha256:abc978a72658f14a2df1e5e12532effe40f94f868f6e23d95133bd6abcca35ca" + "sha256:610c5b90092c360736baccf17bd3efbcb30dd380e7a6dc28a71059edb8bd0d8e", + "sha256:9df18a1f87ee0df0bc4eea2770ebc4228392d8cc4066655b320e2cfccb15db95" ], "markers": "python_version >= '3.7'", - "version": "==2.15.0" + "version": "==2.17.1" }, "google-api-python-client": { "hashes": [ - "sha256:3a45a53c031478d1c82c7162dd25c9a965247bca6bd438af0838a9d9b8219405", - "sha256:b605adee2d09a843b97a59925757802904679e44e5599708cedb8939900dfbc7" + "sha256:9d83b178496b180e058fd206ebfb70ea1afab49f235dd326f557513f56f496d5", + "sha256:ebf4927a3f5184096647be8f705d090e7f06d48ad82b0fa431a2fe80c2cbe182" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==2.111.0" + "version": "==2.118.0" }, "google-auth": { "hashes": [ - "sha256:42f707937feb4f5e5a39e6c4f343a17300a459aaf03141457ba505812841cc40", - "sha256:473a8dfd0135f75bb79d878436e568f2695dce456764bf3a02b6f8c540b1d256" + "sha256:3cfc1b6e4e64797584fb53fc9bd0b7afa9b7c0dba2004fa7dcc9349e58cc3195", + "sha256:7634d29dcd1e101f5226a23cbc4a0c6cda6394253bf80e281d9c5c6797869c53" ], "markers": "python_version >= '3.7'", - "version": "==2.25.2" + "version": "==2.28.0" }, "google-auth-httplib2": { "hashes": [ @@ -683,12 +708,12 @@ }, "gspread": { "hashes": [ - "sha256:298ebab76e6ed6a998eabc81545ec58f5610f44e2ddb4858b539a0634093f8ce", - "sha256:efe5ffa2ff28821452a5964b46f11f72bfb128a6c884ca9e3451f6bb93559e8c" + "sha256:0238ba43f3bd45e7fa96fd206e9ceb73b03c2896eb143d7f4373c6d0cfe6fddf", + "sha256:0982beeb07fa3ec4482a3aaa96ca13a1e6b427a0aca4058beab4cdc33c0cbb64" ], "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==5.12.3" + "markers": "python_version >= '3.8'", + "version": "==6.0.2" }, "h11": { "hashes": [ @@ -700,11 +725,11 @@ }, "httpcore": { "hashes": [ - "sha256:096cc05bca73b8e459a1fc3dcf585148f63e534eae4339559c9b8a8d6399acc7", - "sha256:9fc092e4799b26174648e54b74ed5f683132a464e95643b226e00c2ed2fa6535" + "sha256:5c0f9546ad17dac4d0772b0808856eb616eb8b48ce94f49ed819fd6982a8a544", + "sha256:9a6a501c3099307d9fd76ac244e08503427679b1e81ceb1d922485e2f2462ad2" ], "markers": "python_version >= '3.8'", - "version": "==1.0.2" + "version": "==1.0.3" }, "httplib2": { "hashes": [ @@ -732,11 +757,11 @@ }, "instaloader": { "hashes": [ - "sha256:2ddf1b3e85977bf07141383dff5dab23b2c59ccf40a1d2d8696ad11d43bb8198" + "sha256:168065ab4bc93c1f309e4342883f5645235f2fc17d401125e5c6597d21e2c85b" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==4.10.2" + "version": "==4.10.3" }, "itsdangerous": { "hashes": [ @@ -748,12 +773,12 @@ }, "jinja2": { "hashes": [ - "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", - "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" + "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa", + "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==3.1.2" + "version": "==3.1.3" }, "jmespath": { "hashes": [ @@ -783,101 +808,87 @@ }, "lxml": { "hashes": [ - "sha256:00e91573183ad273e242db5585b52670eddf92bacad095ce25c1e682da14ed91", - "sha256:01bf1df1db327e748dcb152d17389cf6d0a8c5d533ef9bab781e9d5037619229", - "sha256:056a17eaaf3da87a05523472ae84246f87ac2f29a53306466c22e60282e54ff8", - "sha256:0a08c89b23117049ba171bf51d2f9c5f3abf507d65d016d6e0fa2f37e18c0fc5", - "sha256:1343df4e2e6e51182aad12162b23b0a4b3fd77f17527a78c53f0f23573663545", - "sha256:1449f9451cd53e0fd0a7ec2ff5ede4686add13ac7a7bfa6988ff6d75cff3ebe2", - "sha256:16b9ec51cc2feab009e800f2c6327338d6ee4e752c76e95a35c4465e80390ccd", - "sha256:1f10f250430a4caf84115b1e0f23f3615566ca2369d1962f82bef40dd99cd81a", - "sha256:231142459d32779b209aa4b4d460b175cadd604fed856f25c1571a9d78114771", - "sha256:232fd30903d3123be4c435fb5159938c6225ee8607b635a4d3fca847003134ba", - "sha256:23d891e5bdc12e2e506e7d225d6aa929e0a0368c9916c1fddefab88166e98b20", - "sha256:273473d34462ae6e97c0f4e517bd1bf9588aa67a1d47d93f760a1282640e24ac", - "sha256:2bd9ac6e44f2db368ef8986f3989a4cad3de4cd55dbdda536e253000c801bcc7", - "sha256:33714fcf5af4ff7e70a49731a7cc8fd9ce910b9ac194f66eaa18c3cc0a4c02be", - "sha256:359a8b09d712df27849e0bcb62c6a3404e780b274b0b7e4c39a88826d1926c28", - "sha256:365005e8b0718ea6d64b374423e870648ab47c3a905356ab6e5a5ff03962b9a9", - "sha256:389d2b2e543b27962990ab529ac6720c3dded588cc6d0f6557eec153305a3622", - "sha256:3b505f2bbff50d261176e67be24e8909e54b5d9d08b12d4946344066d66b3e43", - "sha256:3d74d4a3c4b8f7a1f676cedf8e84bcc57705a6d7925e6daef7a1e54ae543a197", - "sha256:3f3f00a9061605725df1816f5713d10cd94636347ed651abdbc75828df302b20", - "sha256:43498ea734ccdfb92e1886dfedaebeb81178a241d39a79d5351ba2b671bff2b2", - "sha256:4855161013dfb2b762e02b3f4d4a21cc7c6aec13c69e3bffbf5022b3e708dd97", - "sha256:4d973729ce04784906a19108054e1fd476bc85279a403ea1a72fdb051c76fa48", - "sha256:4ece9cca4cd1c8ba889bfa67eae7f21d0d1a2e715b4d5045395113361e8c533d", - "sha256:506becdf2ecaebaf7f7995f776394fcc8bd8a78022772de66677c84fb02dd33d", - "sha256:520486f27f1d4ce9654154b4494cf9307b495527f3a2908ad4cb48e4f7ed7ef7", - "sha256:5557461f83bb7cc718bc9ee1f7156d50e31747e5b38d79cf40f79ab1447afd2d", - "sha256:562778586949be7e0d7435fcb24aca4810913771f845d99145a6cee64d5b67ca", - "sha256:59bb5979f9941c61e907ee571732219fa4774d5a18f3fa5ff2df963f5dfaa6bc", - "sha256:606d445feeb0856c2b424405236a01c71af7c97e5fe42fbc778634faef2b47e4", - "sha256:6197c3f3c0b960ad033b9b7d611db11285bb461fc6b802c1dd50d04ad715c225", - "sha256:647459b23594f370c1c01768edaa0ba0959afc39caeeb793b43158bb9bb6a663", - "sha256:647bfe88b1997d7ae8d45dabc7c868d8cb0c8412a6e730a7651050b8c7289cf2", - "sha256:6bee9c2e501d835f91460b2c904bc359f8433e96799f5c2ff20feebd9bb1e590", - "sha256:6dbdacf5752fbd78ccdb434698230c4f0f95df7dd956d5f205b5ed6911a1367c", - "sha256:701847a7aaefef121c5c0d855b2affa5f9bd45196ef00266724a80e439220e46", - "sha256:786d6b57026e7e04d184313c1359ac3d68002c33e4b1042ca58c362f1d09ff58", - "sha256:7b378847a09d6bd46047f5f3599cdc64fcb4cc5a5a2dd0a2af610361fbe77b16", - "sha256:7d1d6c9e74c70ddf524e3c09d9dc0522aba9370708c2cb58680ea40174800013", - "sha256:857d6565f9aa3464764c2cb6a2e3c2e75e1970e877c188f4aeae45954a314e0c", - "sha256:8671622256a0859f5089cbe0ce4693c2af407bc053dcc99aadff7f5310b4aa02", - "sha256:88f7c383071981c74ec1998ba9b437659e4fd02a3c4a4d3efc16774eb108d0ec", - "sha256:8aecb5a7f6f7f8fe9cac0bcadd39efaca8bbf8d1bf242e9f175cbe4c925116c3", - "sha256:91bbf398ac8bb7d65a5a52127407c05f75a18d7015a270fdd94bbcb04e65d573", - "sha256:936e8880cc00f839aa4173f94466a8406a96ddce814651075f95837316369899", - "sha256:953dd5481bd6252bd480d6ec431f61d7d87fdcbbb71b0d2bdcfc6ae00bb6fb10", - "sha256:95ae6c5a196e2f239150aa4a479967351df7f44800c93e5a975ec726fef005e2", - "sha256:9a2b5915c333e4364367140443b59f09feae42184459b913f0f41b9fed55794a", - "sha256:9ae6c3363261021144121427b1552b29e7b59de9d6a75bf51e03bc072efb3c37", - "sha256:9b556596c49fa1232b0fff4b0e69b9d4083a502e60e404b44341e2f8fb7187f5", - "sha256:9c131447768ed7bc05a02553d939e7f0e807e533441901dd504e217b76307745", - "sha256:9d9d5726474cbbef279fd709008f91a49c4f758bec9c062dfbba88eab00e3ff9", - "sha256:a1bdcbebd4e13446a14de4dd1825f1e778e099f17f79718b4aeaf2403624b0f7", - "sha256:a602ed9bd2c7d85bd58592c28e101bd9ff9c718fbde06545a70945ffd5d11868", - "sha256:a8edae5253efa75c2fc79a90068fe540b197d1c7ab5803b800fccfe240eed33c", - "sha256:a905affe76f1802edcac554e3ccf68188bea16546071d7583fb1b693f9cf756b", - "sha256:a9e7c6d89c77bb2770c9491d988f26a4b161d05c8ca58f63fb1f1b6b9a74be45", - "sha256:aa9b5abd07f71b081a33115d9758ef6077924082055005808f68feccb27616bd", - "sha256:aaa5c173a26960fe67daa69aa93d6d6a1cd714a6eb13802d4e4bd1d24a530644", - "sha256:ac7674d1638df129d9cb4503d20ffc3922bd463c865ef3cb412f2c926108e9a4", - "sha256:b1541e50b78e15fa06a2670157a1962ef06591d4c998b998047fff5e3236880e", - "sha256:b1980dbcaad634fe78e710c8587383e6e3f61dbe146bcbfd13a9c8ab2d7b1192", - "sha256:bafa65e3acae612a7799ada439bd202403414ebe23f52e5b17f6ffc2eb98c2be", - "sha256:bb5bd6212eb0edfd1e8f254585290ea1dadc3687dd8fd5e2fd9a87c31915cdab", - "sha256:bbdd69e20fe2943b51e2841fc1e6a3c1de460d630f65bde12452d8c97209464d", - "sha256:bc354b1393dce46026ab13075f77b30e40b61b1a53e852e99d3cc5dd1af4bc85", - "sha256:bcee502c649fa6351b44bb014b98c09cb00982a475a1912a9881ca28ab4f9cd9", - "sha256:bdd9abccd0927673cffe601d2c6cdad1c9321bf3437a2f507d6b037ef91ea307", - "sha256:c42ae7e010d7d6bc51875d768110c10e8a59494855c3d4c348b068f5fb81fdcd", - "sha256:c71b5b860c5215fdbaa56f715bc218e45a98477f816b46cfde4a84d25b13274e", - "sha256:c7721a3ef41591341388bb2265395ce522aba52f969d33dacd822da8f018aff8", - "sha256:ca8e44b5ba3edb682ea4e6185b49661fc22b230cf811b9c13963c9f982d1d964", - "sha256:cb53669442895763e61df5c995f0e8361b61662f26c1b04ee82899c2789c8f69", - "sha256:cc02c06e9e320869d7d1bd323df6dd4281e78ac2e7f8526835d3d48c69060683", - "sha256:d3caa09e613ece43ac292fbed513a4bce170681a447d25ffcbc1b647d45a39c5", - "sha256:d82411dbf4d3127b6cde7da0f9373e37ad3a43e89ef374965465928f01c2b979", - "sha256:dbcb2dc07308453db428a95a4d03259bd8caea97d7f0776842299f2d00c72fc8", - "sha256:dd4fda67f5faaef4f9ee5383435048ee3e11ad996901225ad7615bc92245bc8e", - "sha256:ddd92e18b783aeb86ad2132d84a4b795fc5ec612e3545c1b687e7747e66e2b53", - "sha256:de362ac8bc962408ad8fae28f3967ce1a262b5d63ab8cefb42662566737f1dc7", - "sha256:e214025e23db238805a600f1f37bf9f9a15413c7bf5f9d6ae194f84980c78722", - "sha256:e8f9f93a23634cfafbad6e46ad7d09e0f4a25a2400e4a64b1b7b7c0fbaa06d9d", - "sha256:e96a1788f24d03e8d61679f9881a883ecdf9c445a38f9ae3f3f193ab6c591c66", - "sha256:ec53a09aee61d45e7dbe7e91252ff0491b6b5fee3d85b2d45b173d8ab453efc1", - "sha256:f10250bb190fb0742e3e1958dd5c100524c2cc5096c67c8da51233f7448dc137", - "sha256:f1faee2a831fe249e1bae9cbc68d3cd8a30f7e37851deee4d7962b17c410dd56", - "sha256:f610d980e3fccf4394ab3806de6065682982f3d27c12d4ce3ee46a8183d64a6a", - "sha256:f6c35b2f87c004270fa2e703b872fcc984d714d430b305145c39d53074e1ffe0", - "sha256:f836f39678cb47c9541f04d8ed4545719dc31ad850bf1832d6b4171e30d65d23", - "sha256:f99768232f036b4776ce419d3244a04fe83784bce871b16d2c2e984c7fcea847", - "sha256:fd814847901df6e8de13ce69b84c31fc9b3fb591224d6762d0b256d510cbf382", - "sha256:fdb325b7fba1e2c40b9b1db407f85642e32404131c08480dd652110fc908561b" + "sha256:13521a321a25c641b9ea127ef478b580b5ec82aa2e9fc076c86169d161798b01", + "sha256:14deca1460b4b0f6b01f1ddc9557704e8b365f55c63070463f6c18619ebf964f", + "sha256:16018f7099245157564d7148165132c70adb272fb5a17c048ba70d9cc542a1a1", + "sha256:16dd953fb719f0ffc5bc067428fc9e88f599e15723a85618c45847c96f11f431", + "sha256:19a1bc898ae9f06bccb7c3e1dfd73897ecbbd2c96afe9095a6026016e5ca97b8", + "sha256:1ad17c20e3666c035db502c78b86e58ff6b5991906e55bdbef94977700c72623", + "sha256:22b7ee4c35f374e2c20337a95502057964d7e35b996b1c667b5c65c567d2252a", + "sha256:24ef5a4631c0b6cceaf2dbca21687e29725b7c4e171f33a8f8ce23c12558ded1", + "sha256:25663d6e99659544ee8fe1b89b1a8c0aaa5e34b103fab124b17fa958c4a324a6", + "sha256:262bc5f512a66b527d026518507e78c2f9c2bd9eb5c8aeeb9f0eb43fcb69dc67", + "sha256:280f3edf15c2a967d923bcfb1f8f15337ad36f93525828b40a0f9d6c2ad24890", + "sha256:2ad3a8ce9e8a767131061a22cd28fdffa3cd2dc193f399ff7b81777f3520e372", + "sha256:2befa20a13f1a75c751f47e00929fb3433d67eb9923c2c0b364de449121f447c", + "sha256:2f37c6d7106a9d6f0708d4e164b707037b7380fcd0b04c5bd9cae1fb46a856fb", + "sha256:304128394c9c22b6569eba2a6d98392b56fbdfbad58f83ea702530be80d0f9df", + "sha256:342e95bddec3a698ac24378d61996b3ee5ba9acfeb253986002ac53c9a5f6f84", + "sha256:3aeca824b38ca78d9ee2ab82bd9883083d0492d9d17df065ba3b94e88e4d7ee6", + "sha256:3d184e0d5c918cff04cdde9dbdf9600e960161d773666958c9d7b565ccc60c45", + "sha256:3e3898ae2b58eeafedfe99e542a17859017d72d7f6a63de0f04f99c2cb125936", + "sha256:3eea6ed6e6c918e468e693c41ef07f3c3acc310b70ddd9cc72d9ef84bc9564ca", + "sha256:3f14a4fb1c1c402a22e6a341a24c1341b4a3def81b41cd354386dcb795f83897", + "sha256:436a943c2900bb98123b06437cdd30580a61340fbdb7b28aaf345a459c19046a", + "sha256:4946e7f59b7b6a9e27bef34422f645e9a368cb2be11bf1ef3cafc39a1f6ba68d", + "sha256:49a9b4af45e8b925e1cd6f3b15bbba2c81e7dba6dce170c677c9cda547411e14", + "sha256:4f8b0c78e7aac24979ef09b7f50da871c2de2def043d468c4b41f512d831e912", + "sha256:52427a7eadc98f9e62cb1368a5079ae826f94f05755d2d567d93ee1bc3ceb354", + "sha256:5e53d7e6a98b64fe54775d23a7c669763451340c3d44ad5e3a3b48a1efbdc96f", + "sha256:5fcfbebdb0c5d8d18b84118842f31965d59ee3e66996ac842e21f957eb76138c", + "sha256:601f4a75797d7a770daed8b42b97cd1bb1ba18bd51a9382077a6a247a12aa38d", + "sha256:61c5a7edbd7c695e54fca029ceb351fc45cd8860119a0f83e48be44e1c464862", + "sha256:6a2a2c724d97c1eb8cf966b16ca2915566a4904b9aad2ed9a09c748ffe14f969", + "sha256:6d48fc57e7c1e3df57be5ae8614bab6d4e7b60f65c5457915c26892c41afc59e", + "sha256:6f11b77ec0979f7e4dc5ae081325a2946f1fe424148d3945f943ceaede98adb8", + "sha256:704f5572ff473a5f897745abebc6df40f22d4133c1e0a1f124e4f2bd3330ff7e", + "sha256:725e171e0b99a66ec8605ac77fa12239dbe061482ac854d25720e2294652eeaa", + "sha256:7cfced4a069003d8913408e10ca8ed092c49a7f6cefee9bb74b6b3e860683b45", + "sha256:7ec465e6549ed97e9f1e5ed51c657c9ede767bc1c11552f7f4d022c4df4a977a", + "sha256:82bddf0e72cb2af3cbba7cec1d2fd11fda0de6be8f4492223d4a268713ef2147", + "sha256:82cd34f1081ae4ea2ede3d52f71b7be313756e99b4b5f829f89b12da552d3aa3", + "sha256:843b9c835580d52828d8f69ea4302537337a21e6b4f1ec711a52241ba4a824f3", + "sha256:877efb968c3d7eb2dad540b6cabf2f1d3c0fbf4b2d309a3c141f79c7e0061324", + "sha256:8b9f19df998761babaa7f09e6bc169294eefafd6149aaa272081cbddc7ba4ca3", + "sha256:8cf5877f7ed384dabfdcc37922c3191bf27e55b498fecece9fd5c2c7aaa34c33", + "sha256:8d2900b7f5318bc7ad8631d3d40190b95ef2aa8cc59473b73b294e4a55e9f30f", + "sha256:8d7b4beebb178e9183138f552238f7e6613162a42164233e2bda00cb3afac58f", + "sha256:8f52fe6859b9db71ee609b0c0a70fea5f1e71c3462ecf144ca800d3f434f0764", + "sha256:98f3f020a2b736566c707c8e034945c02aa94e124c24f77ca097c446f81b01f1", + "sha256:9aa543980ab1fbf1720969af1d99095a548ea42e00361e727c58a40832439114", + "sha256:9b99f564659cfa704a2dd82d0684207b1aadf7d02d33e54845f9fc78e06b7581", + "sha256:9bcf86dfc8ff3e992fed847c077bd875d9e0ba2fa25d859c3a0f0f76f07f0c8d", + "sha256:9bd0ae7cc2b85320abd5e0abad5ccee5564ed5f0cc90245d2f9a8ef330a8deae", + "sha256:9d3c0f8567ffe7502d969c2c1b809892dc793b5d0665f602aad19895f8d508da", + "sha256:9e5ac3437746189a9b4121db2a7b86056ac8786b12e88838696899328fc44bb2", + "sha256:a36c506e5f8aeb40680491d39ed94670487ce6614b9d27cabe45d94cd5d63e1e", + "sha256:a5ab722ae5a873d8dcee1f5f45ddd93c34210aed44ff2dc643b5025981908cda", + "sha256:a96f02ba1bcd330807fc060ed91d1f7a20853da6dd449e5da4b09bfcc08fdcf5", + "sha256:acb6b2f96f60f70e7f34efe0c3ea34ca63f19ca63ce90019c6cbca6b676e81fa", + "sha256:ae15347a88cf8af0949a9872b57a320d2605ae069bcdf047677318bc0bba45b1", + "sha256:af8920ce4a55ff41167ddbc20077f5698c2e710ad3353d32a07d3264f3a2021e", + "sha256:afd825e30f8d1f521713a5669b63657bcfe5980a916c95855060048b88e1adb7", + "sha256:b21b4031b53d25b0858d4e124f2f9131ffc1530431c6d1321805c90da78388d1", + "sha256:b4b68c961b5cc402cbd99cca5eb2547e46ce77260eb705f4d117fd9c3f932b95", + "sha256:b66aa6357b265670bb574f050ffceefb98549c721cf28351b748be1ef9577d93", + "sha256:b9e240ae0ba96477682aa87899d94ddec1cc7926f9df29b1dd57b39e797d5ab5", + "sha256:bc64d1b1dab08f679fb89c368f4c05693f58a9faf744c4d390d7ed1d8223869b", + "sha256:bf8443781533b8d37b295016a4b53c1494fa9a03573c09ca5104550c138d5c05", + "sha256:c26aab6ea9c54d3bed716b8851c8bfc40cb249b8e9880e250d1eddde9f709bf5", + "sha256:c3cd1fc1dc7c376c54440aeaaa0dcc803d2126732ff5c6b68ccd619f2e64be4f", + "sha256:c7257171bb8d4432fe9d6fdde4d55fdbe663a63636a17f7f9aaba9bcb3153ad7", + "sha256:d42e3a3fc18acc88b838efded0e6ec3edf3e328a58c68fbd36a7263a874906c8", + "sha256:d74fcaf87132ffc0447b3c685a9f862ffb5b43e70ea6beec2fb8057d5d2a1fea", + "sha256:d8c1d679df4361408b628f42b26a5d62bd3e9ba7f0c0e7969f925021554755aa", + "sha256:e856c1c7255c739434489ec9c8aa9cdf5179785d10ff20add308b5d673bed5cd", + "sha256:eac68f96539b32fce2c9b47eb7c25bb2582bdaf1bbb360d25f564ee9e04c542b", + "sha256:ed7326563024b6e91fef6b6c7a1a2ff0a71b97793ac33dbbcf38f6005e51ff6e", + "sha256:ed8c3d2cd329bf779b7ed38db176738f3f8be637bb395ce9629fc76f78afe3d4", + "sha256:f4c9bda132ad108b387c33fabfea47866af87f4ea6ffb79418004f0521e63204", + "sha256:f643ffd2669ffd4b5a3e9b41c909b72b2a1d5e4915da90a77e119b8d48ce867a" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==4.9.4" + "markers": "python_version >= '3.6'", + "version": "==5.1.0" }, "markdown-it-py": { "hashes": [ @@ -889,77 +900,77 @@ }, "markupsafe": { "hashes": [ - "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e", - "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e", - "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431", - "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686", - "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c", - "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559", - "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc", - "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb", - "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939", - "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c", - "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0", - "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4", - "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9", - "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575", - "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba", - "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d", - "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd", - "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3", - "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00", - "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155", - "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac", - "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52", - "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f", - "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8", - "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b", - "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007", - "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24", - "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea", - "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198", - "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0", - "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee", - "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be", - "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2", - "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1", - "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707", - "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6", - "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c", - "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58", - "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823", - "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779", - "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636", - "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c", - "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad", - "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee", - "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc", - "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2", - "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48", - "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7", - "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e", - "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b", - "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa", - "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5", - "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e", - "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb", - "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9", - "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57", - "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc", - "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc", - "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2", - "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11" + "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", + "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff", + "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f", + "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3", + "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532", + "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f", + "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617", + "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df", + "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4", + "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906", + "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f", + "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4", + "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8", + "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371", + "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2", + "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465", + "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52", + "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6", + "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169", + "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad", + "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2", + "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0", + "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029", + "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f", + "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a", + "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced", + "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5", + "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c", + "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf", + "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9", + "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb", + "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad", + "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3", + "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1", + "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46", + "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc", + "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a", + "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee", + "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900", + "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5", + "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea", + "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f", + "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5", + "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e", + "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a", + "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f", + "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50", + "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a", + "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b", + "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4", + "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff", + "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2", + "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46", + "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b", + "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf", + "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5", + "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5", + "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", + "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd", + "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68" ], "markers": "python_version >= '3.7'", - "version": "==2.1.3" + "version": "==2.1.5" }, "marshmallow": { "hashes": [ - "sha256:5d2371bbe42000f2b3fb5eaa065224df7d8f8597bc19a1bbfa5bfe7fba8da889", - "sha256:684939db93e80ad3561392f47be0230743131560a41c5110684c16e21ade0a5c" + "sha256:4c1daff273513dc5eb24b219a8035559dc573c8f322558ef85f5438ddd1236dd", + "sha256:c21d4b98fee747c130e6bc8f45c4b3199ea66bc00c12ee1f639f0aeca034d5e9" ], "markers": "python_version >= '3.8'", - "version": "==3.20.1" + "version": "==3.20.2" }, "mdurl": { "hashes": [ @@ -969,85 +980,133 @@ "markers": "python_version >= '3.7'", "version": "==0.1.2" }, + "minify-html": { + "hashes": [ + "sha256:01ea40dc5ae073c47024f02758d5e18e55d853265eb9c099040a6c00ab0abb99", + "sha256:1056819ea46e9080db6fed678d03511c7e94c2a615e72df82190ea898dc82609", + "sha256:2a9aef71b24c3d38c6bece2db3bf707443894958b01f1c27d3a6459ba4200e59", + "sha256:3b38ea5b446cc69e691a0bf64d1160332ffc220bb5b411775983c87311cab2c7", + "sha256:40f38ddfefbb63beb28df20c2c81c12e6af6838387520506b4eceec807d794a3", + "sha256:597c86f9792437eee0698118fb38dff42b5b4be6d437b6d577453c2f91524ccc", + "sha256:5f707b233b9c163a546b15ce9af433ddd456bd113f0326e5ffb382b8ee5c1a2d", + "sha256:70251bd7174b62c91333110301b27000b547aa2cc06d4fe6ba6c3f11612eecc9", + "sha256:7a5eb7e830277762da69498ee0f15d4a9fa6e91887a93567d388e4f5aee01ec3", + "sha256:7af72438d3ae6ea8b0a94c038d35c9c22c5f8540967f5fa2487f77b2cdb12605", + "sha256:7b071ded7aacbb140a7e751d49e246052f204b896d69663a4a5c3a27203d27f6", + "sha256:7b2aadba6987e6c15a916a4627b94b1db3cbac65e6ae3613b61b3ab0d2bb4c96", + "sha256:7e6d4f97cebb725bc1075f225bdfcd824e0f5c20a37d9ea798d900f96e1b80c0", + "sha256:92375f0cb3b4074e45005e1b4708b5b4c0781b335659d52918671c083c19c71e", + "sha256:a23a8055e65fa01175ddd7d18d101c05e267410fa5956c65597dcc332c7f91dd", + "sha256:afd76ca2dc9afa53b66973a3a66eff9a64692811ead44102aa8044a37872e6e2", + "sha256:b6356541799951c5e8205aabf5970dda687f4ffa736479ce8df031919861e51d", + "sha256:bd682207673246c78fb895e7065425cc94cb712d94cff816dd9752ce014f23e8", + "sha256:cda674cc68ec3b9ebf61f2986f3ef62de60ce837a58860c6f16b011862b5d533", + "sha256:cf4c36b6f9af3b0901bd2a0a29db3b09c0cdf0c38d3dde28e6835bce0f605d37", + "sha256:d4c4ae3909e2896c865ebaa3a96939191f904dd337a87d7594130f3dfca55510", + "sha256:dc2df1e5203d89197f530d14c9a82067f3d04b9cb0118abc8f2ef8f88efce109", + "sha256:e47197849a1c09a95892d32df3c9e15f6d0902c9ae215e73249b9f5bca9aeb97", + "sha256:ea315ad6ac33d7463fac3f313bba8c8d9a55f4811971c203eed931203047e5c8", + "sha256:ef6dc1950e04b7566c1ece72712674416f86fef8966ca026f6c5580d840cd354", + "sha256:f37ce536305500914fd4ee2bbaa4dd05a039f39eeceae45560c39767d99aede0" + ], + "index": "pypi", + "version": "==0.15.0" + }, "multidict": { "hashes": [ - "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9", - "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8", - "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03", - "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710", - "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161", - "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664", - "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569", - "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067", - "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313", - "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706", - "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2", - "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636", - "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49", - "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93", - "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603", - "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0", - "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60", - "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4", - "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e", - "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1", - "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60", - "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951", - "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc", - "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe", - "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95", - "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d", - "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8", - "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed", - "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2", - "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775", - "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87", - "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c", - "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2", - "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98", - "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3", - "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe", - "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78", - "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660", - "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176", - "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e", - "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988", - "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c", - "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c", - "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0", - "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449", - "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f", - "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde", - "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5", - "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d", - "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac", - "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a", - "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9", - "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca", - "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11", - "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35", - "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063", - "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b", - "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982", - "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258", - "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1", - "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52", - "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480", - "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7", - "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461", - "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d", - "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc", - "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779", - "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a", - "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547", - "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0", - "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171", - "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf", - "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d", - "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba" + "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556", + "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c", + "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29", + "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b", + "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8", + "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7", + "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd", + "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40", + "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6", + "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3", + "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c", + "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9", + "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5", + "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae", + "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442", + "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9", + "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc", + "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c", + "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea", + "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5", + "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50", + "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182", + "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453", + "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e", + "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600", + "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733", + "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda", + "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241", + "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461", + "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e", + "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e", + "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b", + "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e", + "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7", + "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386", + "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd", + "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9", + "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf", + "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee", + "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5", + "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a", + "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271", + "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54", + "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4", + "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496", + "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb", + "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319", + "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3", + "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f", + "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527", + "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed", + "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604", + "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef", + "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8", + "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5", + "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5", + "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626", + "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c", + "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d", + "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c", + "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc", + "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc", + "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b", + "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38", + "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450", + "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1", + "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f", + "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3", + "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755", + "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226", + "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a", + "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046", + "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf", + "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479", + "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e", + "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1", + "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a", + "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83", + "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929", + "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93", + "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a", + "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c", + "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44", + "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89", + "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba", + "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e", + "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da", + "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24", + "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423", + "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef" ], "markers": "python_version >= '3.7'", - "version": "==6.0.4" + "version": "==6.0.5" }, "mutagen": { "hashes": [ @@ -1067,46 +1126,46 @@ }, "numpy": { "hashes": [ - "sha256:06fa1ed84aa60ea6ef9f91ba57b5ed963c3729534e6e54055fc151fad0423f0a", - "sha256:174a8880739c16c925799c018f3f55b8130c1f7c8e75ab0a6fa9d41cab092fd6", - "sha256:1a13860fdcd95de7cf58bd6f8bc5a5ef81c0b0625eb2c9a783948847abbef2c2", - "sha256:1cc3d5029a30fb5f06704ad6b23b35e11309491c999838c31f124fee32107c79", - "sha256:22f8fc02fdbc829e7a8c578dd8d2e15a9074b630d4da29cda483337e300e3ee9", - "sha256:26c9d33f8e8b846d5a65dd068c14e04018d05533b348d9eaeef6c1bd787f9919", - "sha256:2b3fca8a5b00184828d12b073af4d0fc5fdd94b1632c2477526f6bd7842d700d", - "sha256:2beef57fb031dcc0dc8fa4fe297a742027b954949cabb52a2a376c144e5e6060", - "sha256:36340109af8da8805d8851ef1d74761b3b88e81a9bd80b290bbfed61bd2b4f75", - "sha256:3703fc9258a4a122d17043e57b35e5ef1c5a5837c3db8be396c82e04c1cf9b0f", - "sha256:3ced40d4e9e18242f70dd02d739e44698df3dcb010d31f495ff00a31ef6014fe", - "sha256:4a06263321dfd3598cacb252f51e521a8cb4b6df471bb12a7ee5cbab20ea9167", - "sha256:4eb8df4bf8d3d90d091e0146f6c28492b0be84da3e409ebef54349f71ed271ef", - "sha256:5d5244aabd6ed7f312268b9247be47343a654ebea52a60f002dc70c769048e75", - "sha256:64308ebc366a8ed63fd0bf426b6a9468060962f1a4339ab1074c228fa6ade8e3", - "sha256:6a3cdb4d9c70e6b8c0814239ead47da00934666f668426fc6e94cce869e13fd7", - "sha256:854ab91a2906ef29dc3925a064fcd365c7b4da743f84b123002f6139bcb3f8a7", - "sha256:94cc3c222bb9fb5a12e334d0479b97bb2df446fbe622b470928f5284ffca3f8d", - "sha256:96ca5482c3dbdd051bcd1fce8034603d6ebfc125a7bd59f55b40d8f5d246832b", - "sha256:a2bbc29fcb1771cd7b7425f98b05307776a6baf43035d3b80c4b0f29e9545186", - "sha256:a4cd6ed4a339c21f1d1b0fdf13426cb3b284555c27ac2f156dfdaaa7e16bfab0", - "sha256:aa18428111fb9a591d7a9cc1b48150097ba6a7e8299fb56bdf574df650e7d1f1", - "sha256:aa317b2325f7aa0a9471663e6093c210cb2ae9c0ad824732b307d2c51983d5b6", - "sha256:b04f5dc6b3efdaab541f7857351aac359e6ae3c126e2edb376929bd3b7f92d7e", - "sha256:b272d4cecc32c9e19911891446b72e986157e6a1809b7b56518b4f3755267523", - "sha256:b361d369fc7e5e1714cf827b731ca32bff8d411212fccd29ad98ad622449cc36", - "sha256:b96e7b9c624ef3ae2ae0e04fa9b460f6b9f17ad8b4bec6d7756510f1f6c0c841", - "sha256:baf8aab04a2c0e859da118f0b38617e5ee65d75b83795055fb66c0d5e9e9b818", - "sha256:bcc008217145b3d77abd3e4d5ef586e3bdfba8fe17940769f8aa09b99e856c00", - "sha256:bd3f0091e845164a20bd5a326860c840fe2af79fa12e0469a12768a3ec578d80", - "sha256:cc392fdcbd21d4be6ae1bb4475a03ce3b025cd49a9be5345d76d7585aea69440", - "sha256:d73a3abcac238250091b11caef9ad12413dab01669511779bc9b29261dd50210", - "sha256:f43740ab089277d403aa07567be138fc2a89d4d9892d113b76153e0e412409f8", - "sha256:f65738447676ab5777f11e6bbbdb8ce11b785e105f690bc45966574816b6d3ea", - "sha256:f79b231bf5c16b1f39c7f4875e1ded36abee1591e98742b05d8a0fb55d8a3eec", - "sha256:fe6b44fb8fcdf7eda4ef4461b97b3f63c466b27ab151bec2366db8b197387841" + "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", + "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", + "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20", + "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", + "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", + "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a", + "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea", + "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c", + "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71", + "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", + "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be", + "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a", + "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a", + "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5", + "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", + "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd", + "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c", + "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e", + "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0", + "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c", + "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a", + "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", + "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0", + "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6", + "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2", + "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", + "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30", + "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", + "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5", + "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07", + "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2", + "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4", + "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764", + "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef", + "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3", + "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==1.26.2" + "version": "==1.26.4" }, "oauth2client": { "hashes": [ @@ -1124,6 +1183,13 @@ "markers": "python_version >= '3.6'", "version": "==3.2.2" }, + "oscrypto": { + "hashes": [ + "sha256:2b2f1d2d42ec152ca90ccb5682f3e051fb55986e1b170ebde472b133713e7085", + "sha256:6f5fef59cb5b3708321db7cca56aed8ad7e662853351e7991fcf60ec606d47a4" + ], + "version": "==1.3.0" + }, "outcome": { "hashes": [ "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8", @@ -1157,81 +1223,95 @@ }, "pillow": { "hashes": [ - "sha256:00f438bb841382b15d7deb9a05cc946ee0f2c352653c7aa659e75e592f6fa17d", - "sha256:0248f86b3ea061e67817c47ecbe82c23f9dd5d5226200eb9090b3873d3ca32de", - "sha256:04f6f6149f266a100374ca3cc368b67fb27c4af9f1cc8cb6306d849dcdf12616", - "sha256:062a1610e3bc258bff2328ec43f34244fcec972ee0717200cb1425214fe5b839", - "sha256:0a026c188be3b443916179f5d04548092e253beb0c3e2ee0a4e2cdad72f66099", - "sha256:0f7c276c05a9767e877a0b4c5050c8bee6a6d960d7f0c11ebda6b99746068c2a", - "sha256:1a8413794b4ad9719346cd9306118450b7b00d9a15846451549314a58ac42219", - "sha256:1ab05f3db77e98f93964697c8efc49c7954b08dd61cff526b7f2531a22410106", - "sha256:1c3ac5423c8c1da5928aa12c6e258921956757d976405e9467c5f39d1d577a4b", - "sha256:1c41d960babf951e01a49c9746f92c5a7e0d939d1652d7ba30f6b3090f27e412", - "sha256:1fafabe50a6977ac70dfe829b2d5735fd54e190ab55259ec8aea4aaea412fa0b", - "sha256:1fb29c07478e6c06a46b867e43b0bcdb241b44cc52be9bc25ce5944eed4648e7", - "sha256:24fadc71218ad2b8ffe437b54876c9382b4a29e030a05a9879f615091f42ffc2", - "sha256:2cdc65a46e74514ce742c2013cd4a2d12e8553e3a2563c64879f7c7e4d28bce7", - "sha256:2ef6721c97894a7aa77723740a09547197533146fba8355e86d6d9a4a1056b14", - "sha256:3b834f4b16173e5b92ab6566f0473bfb09f939ba14b23b8da1f54fa63e4b623f", - "sha256:3d929a19f5469b3f4df33a3df2983db070ebb2088a1e145e18facbc28cae5b27", - "sha256:41f67248d92a5e0a2076d3517d8d4b1e41a97e2df10eb8f93106c89107f38b57", - "sha256:47e5bf85b80abc03be7455c95b6d6e4896a62f6541c1f2ce77a7d2bb832af262", - "sha256:4d0152565c6aa6ebbfb1e5d8624140a440f2b99bf7afaafbdbf6430426497f28", - "sha256:50d08cd0a2ecd2a8657bd3d82c71efd5a58edb04d9308185d66c3a5a5bed9610", - "sha256:61f1a9d247317fa08a308daaa8ee7b3f760ab1809ca2da14ecc88ae4257d6172", - "sha256:6932a7652464746fcb484f7fc3618e6503d2066d853f68a4bd97193a3996e273", - "sha256:7a7e3daa202beb61821c06d2517428e8e7c1aab08943e92ec9e5755c2fc9ba5e", - "sha256:7dbaa3c7de82ef37e7708521be41db5565004258ca76945ad74a8e998c30af8d", - "sha256:7df5608bc38bd37ef585ae9c38c9cd46d7c81498f086915b0f97255ea60c2818", - "sha256:806abdd8249ba3953c33742506fe414880bad78ac25cc9a9b1c6ae97bedd573f", - "sha256:883f216eac8712b83a63f41b76ddfb7b2afab1b74abbb413c5df6680f071a6b9", - "sha256:912e3812a1dbbc834da2b32299b124b5ddcb664ed354916fd1ed6f193f0e2d01", - "sha256:937bdc5a7f5343d1c97dc98149a0be7eb9704e937fe3dc7140e229ae4fc572a7", - "sha256:9882a7451c680c12f232a422730f986a1fcd808da0fd428f08b671237237d651", - "sha256:9a92109192b360634a4489c0c756364c0c3a2992906752165ecb50544c251312", - "sha256:9d7bc666bd8c5a4225e7ac71f2f9d12466ec555e89092728ea0f5c0c2422ea80", - "sha256:a5f63b5a68daedc54c7c3464508d8c12075e56dcfbd42f8c1bf40169061ae666", - "sha256:a646e48de237d860c36e0db37ecaecaa3619e6f3e9d5319e527ccbc8151df061", - "sha256:a89b8312d51715b510a4fe9fc13686283f376cfd5abca8cd1c65e4c76e21081b", - "sha256:a92386125e9ee90381c3369f57a2a50fa9e6aa8b1cf1d9c4b200d41a7dd8e992", - "sha256:ae88931f93214777c7a3aa0a8f92a683f83ecde27f65a45f95f22d289a69e593", - "sha256:afc8eef765d948543a4775f00b7b8c079b3321d6b675dde0d02afa2ee23000b4", - "sha256:b0eb01ca85b2361b09480784a7931fc648ed8b7836f01fb9241141b968feb1db", - "sha256:b1c25762197144e211efb5f4e8ad656f36c8d214d390585d1d21281f46d556ba", - "sha256:b4005fee46ed9be0b8fb42be0c20e79411533d1fd58edabebc0dd24626882cfd", - "sha256:b920e4d028f6442bea9a75b7491c063f0b9a3972520731ed26c83e254302eb1e", - "sha256:baada14941c83079bf84c037e2d8b7506ce201e92e3d2fa0d1303507a8538212", - "sha256:bb40c011447712d2e19cc261c82655f75f32cb724788df315ed992a4d65696bb", - "sha256:c0949b55eb607898e28eaccb525ab104b2d86542a85c74baf3a6dc24002edec2", - "sha256:c9aeea7b63edb7884b031a35305629a7593272b54f429a9869a4f63a1bf04c34", - "sha256:cfe96560c6ce2f4c07d6647af2d0f3c54cc33289894ebd88cfbb3bcd5391e256", - "sha256:d27b5997bdd2eb9fb199982bb7eb6164db0426904020dc38c10203187ae2ff2f", - "sha256:d921bc90b1defa55c9917ca6b6b71430e4286fc9e44c55ead78ca1a9f9eba5f2", - "sha256:e6bf8de6c36ed96c86ea3b6e1d5273c53f46ef518a062464cd7ef5dd2cf92e38", - "sha256:eaed6977fa73408b7b8a24e8b14e59e1668cfc0f4c40193ea7ced8e210adf996", - "sha256:fa1d323703cfdac2036af05191b969b910d8f115cf53093125e4058f62012c9a", - "sha256:fe1e26e1ffc38be097f0ba1d0d07fcade2bcfd1d023cda5b29935ae8052bd793" + "sha256:0304004f8067386b477d20a518b50f3fa658a28d44e4116970abfcd94fac34a8", + "sha256:0689b5a8c5288bc0504d9fcee48f61a6a586b9b98514d7d29b840143d6734f39", + "sha256:0eae2073305f451d8ecacb5474997c08569fb4eb4ac231ffa4ad7d342fdc25ac", + "sha256:0fb3e7fc88a14eacd303e90481ad983fd5b69c761e9e6ef94c983f91025da869", + "sha256:11fa2e5984b949b0dd6d7a94d967743d87c577ff0b83392f17cb3990d0d2fd6e", + "sha256:127cee571038f252a552760076407f9cff79761c3d436a12af6000cd182a9d04", + "sha256:154e939c5f0053a383de4fd3d3da48d9427a7e985f58af8e94d0b3c9fcfcf4f9", + "sha256:15587643b9e5eb26c48e49a7b33659790d28f190fc514a322d55da2fb5c2950e", + "sha256:170aeb00224ab3dc54230c797f8404507240dd868cf52066f66a41b33169bdbe", + "sha256:1b5e1b74d1bd1b78bc3477528919414874748dd363e6272efd5abf7654e68bef", + "sha256:1da3b2703afd040cf65ec97efea81cfba59cdbed9c11d8efc5ab09df9509fc56", + "sha256:1e23412b5c41e58cec602f1135c57dfcf15482013ce6e5f093a86db69646a5aa", + "sha256:2247178effb34a77c11c0e8ac355c7a741ceca0a732b27bf11e747bbc950722f", + "sha256:257d8788df5ca62c980314053197f4d46eefedf4e6175bc9412f14412ec4ea2f", + "sha256:3031709084b6e7852d00479fd1d310b07d0ba82765f973b543c8af5061cf990e", + "sha256:322209c642aabdd6207517e9739c704dc9f9db943015535783239022002f054a", + "sha256:322bdf3c9b556e9ffb18f93462e5f749d3444ce081290352c6070d014c93feb2", + "sha256:33870dc4653c5017bf4c8873e5488d8f8d5f8935e2f1fb9a2208c47cdd66efd2", + "sha256:35bb52c37f256f662abdfa49d2dfa6ce5d93281d323a9af377a120e89a9eafb5", + "sha256:3c31822339516fb3c82d03f30e22b1d038da87ef27b6a78c9549888f8ceda39a", + "sha256:3eedd52442c0a5ff4f887fab0c1c0bb164d8635b32c894bc1faf4c618dd89df2", + "sha256:3ff074fc97dd4e80543a3e91f69d58889baf2002b6be64347ea8cf5533188213", + "sha256:47c0995fc4e7f79b5cfcab1fc437ff2890b770440f7696a3ba065ee0fd496563", + "sha256:49d9ba1ed0ef3e061088cd1e7538a0759aab559e2e0a80a36f9fd9d8c0c21591", + "sha256:51f1a1bffc50e2e9492e87d8e09a17c5eea8409cda8d3f277eb6edc82813c17c", + "sha256:52a50aa3fb3acb9cf7213573ef55d31d6eca37f5709c69e6858fe3bc04a5c2a2", + "sha256:54f1852cd531aa981bc0965b7d609f5f6cc8ce8c41b1139f6ed6b3c54ab82bfb", + "sha256:609448742444d9290fd687940ac0b57fb35e6fd92bdb65386e08e99af60bf757", + "sha256:69ffdd6120a4737710a9eee73e1d2e37db89b620f702754b8f6e62594471dee0", + "sha256:6fad5ff2f13d69b7e74ce5b4ecd12cc0ec530fcee76356cac6742785ff71c452", + "sha256:7049e301399273a0136ff39b84c3678e314f2158f50f517bc50285fb5ec847ad", + "sha256:70c61d4c475835a19b3a5aa42492409878bbca7438554a1f89d20d58a7c75c01", + "sha256:716d30ed977be8b37d3ef185fecb9e5a1d62d110dfbdcd1e2a122ab46fddb03f", + "sha256:753cd8f2086b2b80180d9b3010dd4ed147efc167c90d3bf593fe2af21265e5a5", + "sha256:773efe0603db30c281521a7c0214cad7836c03b8ccff897beae9b47c0b657d61", + "sha256:7823bdd049099efa16e4246bdf15e5a13dbb18a51b68fa06d6c1d4d8b99a796e", + "sha256:7c8f97e8e7a9009bcacbe3766a36175056c12f9a44e6e6f2d5caad06dcfbf03b", + "sha256:823ef7a27cf86df6597fa0671066c1b596f69eba53efa3d1e1cb8b30f3533068", + "sha256:8373c6c251f7ef8bda6675dd6d2b3a0fcc31edf1201266b5cf608b62a37407f9", + "sha256:83b2021f2ade7d1ed556bc50a399127d7fb245e725aa0113ebd05cfe88aaf588", + "sha256:870ea1ada0899fd0b79643990809323b389d4d1d46c192f97342eeb6ee0b8483", + "sha256:8d12251f02d69d8310b046e82572ed486685c38f02176bd08baf216746eb947f", + "sha256:9c23f307202661071d94b5e384e1e1dc7dfb972a28a2310e4ee16103e66ddb67", + "sha256:9d189550615b4948f45252d7f005e53c2040cea1af5b60d6f79491a6e147eef7", + "sha256:a086c2af425c5f62a65e12fbf385f7c9fcb8f107d0849dba5839461a129cf311", + "sha256:a2b56ba36e05f973d450582fb015594aaa78834fefe8dfb8fcd79b93e64ba4c6", + "sha256:aebb6044806f2e16ecc07b2a2637ee1ef67a11840a66752751714a0d924adf72", + "sha256:b1b3020d90c2d8e1dae29cf3ce54f8094f7938460fb5ce8bc5c01450b01fbaf6", + "sha256:b4b6b1e20608493548b1f32bce8cca185bf0480983890403d3b8753e44077129", + "sha256:b6f491cdf80ae540738859d9766783e3b3c8e5bd37f5dfa0b76abdecc5081f13", + "sha256:b792a349405fbc0163190fde0dc7b3fef3c9268292586cf5645598b48e63dc67", + "sha256:b7c2286c23cd350b80d2fc9d424fc797575fb16f854b831d16fd47ceec078f2c", + "sha256:babf5acfede515f176833ed6028754cbcd0d206f7f614ea3447d67c33be12516", + "sha256:c365fd1703040de1ec284b176d6af5abe21b427cb3a5ff68e0759e1e313a5e7e", + "sha256:c4225f5220f46b2fde568c74fca27ae9771536c2e29d7c04f4fb62c83275ac4e", + "sha256:c570f24be1e468e3f0ce7ef56a89a60f0e05b30a3669a459e419c6eac2c35364", + "sha256:c6dafac9e0f2b3c78df97e79af707cdc5ef8e88208d686a4847bab8266870023", + "sha256:c8de2789052ed501dd829e9cae8d3dcce7acb4777ea4a479c14521c942d395b1", + "sha256:cb28c753fd5eb3dd859b4ee95de66cc62af91bcff5db5f2571d32a520baf1f04", + "sha256:cb4c38abeef13c61d6916f264d4845fab99d7b711be96c326b84df9e3e0ff62d", + "sha256:d1b35bcd6c5543b9cb547dee3150c93008f8dd0f1fef78fc0cd2b141c5baf58a", + "sha256:d8e6aeb9201e655354b3ad049cb77d19813ad4ece0df1249d3c793de3774f8c7", + "sha256:d8ecd059fdaf60c1963c58ceb8997b32e9dc1b911f5da5307aab614f1ce5c2fb", + "sha256:da2b52b37dad6d9ec64e653637a096905b258d2fc2b984c41ae7d08b938a67e4", + "sha256:e87f0b2c78157e12d7686b27d63c070fd65d994e8ddae6f328e0dcf4a0cd007e", + "sha256:edca80cbfb2b68d7b56930b84a0e45ae1694aeba0541f798e908a49d66b837f1", + "sha256:f379abd2f1e3dddb2b61bc67977a6b5a0a3f7485538bcc6f39ec76163891ee48", + "sha256:fe4c15f6c9285dc54ce6553a3ce908ed37c8f3825b5a51a15c91442bb955b868" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==10.1.0" + "version": "==10.2.0" }, "protobuf": { "hashes": [ - "sha256:0bf384e75b92c42830c0a679b0cd4d6e2b36ae0cf3dbb1e1dfdda48a244f4bcd", - "sha256:0f881b589ff449bf0b931a711926e9ddaad3b35089cc039ce1af50b21a4ae8cb", - "sha256:1484f9e692091450e7edf418c939e15bfc8fc68856e36ce399aed6889dae8bb0", - "sha256:193f50a6ab78a970c9b4f148e7c750cfde64f59815e86f686c22e26b4fe01ce7", - "sha256:3497c1af9f2526962f09329fd61a36566305e6c72da2590ae0d7d1322818843b", - "sha256:57d65074b4f5baa4ab5da1605c02be90ac20c8b40fb137d6a8df9f416b0d0ce2", - "sha256:8bdbeaddaac52d15c6dce38c71b03038ef7772b977847eb6d374fc86636fa510", - "sha256:a19731d5e83ae4737bb2a089605e636077ac001d18781b3cf489b9546c7c80d6", - "sha256:abc0525ae2689a8000837729eef7883b9391cd6aa7950249dcf5a4ede230d5dd", - "sha256:becc576b7e6b553d22cbdf418686ee4daa443d7217999125c045ad56322dda10", - "sha256:ca37bf6a6d0046272c152eea90d2e4ef34593aaa32e8873fc14c16440f22d4b7" + "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4", + "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8", + "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c", + "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d", + "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4", + "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa", + "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c", + "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019", + "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9", + "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c", + "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2" ], "markers": "python_version >= '3.8'", - "version": "==4.25.1" + "version": "==4.25.3" }, "pyaes": { "hashes": [ @@ -1264,41 +1344,41 @@ }, "pycryptodomex": { "hashes": [ - "sha256:09c9401dc06fb3d94cb1ec23b4ea067a25d1f4c6b7b118ff5631d0b5daaab3cc", - "sha256:0b2f1982c5bc311f0aab8c293524b861b485d76f7c9ab2c3ac9a25b6f7655975", - "sha256:136b284e9246b4ccf4f752d435c80f2c44fc2321c198505de1d43a95a3453b3c", - "sha256:1789d89f61f70a4cd5483d4dfa8df7032efab1118f8b9894faae03c967707865", - "sha256:2126bc54beccbede6eade00e647106b4f4c21e5201d2b0a73e9e816a01c50905", - "sha256:258c4233a3fe5a6341780306a36c6fb072ef38ce676a6d41eec3e591347919e8", - "sha256:263de9a96d2fcbc9f5bd3a279f14ea0d5f072adb68ebd324987576ec25da084d", - "sha256:50cb18d4dd87571006fd2447ccec85e6cec0136632a550aa29226ba075c80644", - "sha256:5b883e1439ab63af976656446fb4839d566bb096f15fc3c06b5a99cde4927188", - "sha256:5d73e9fa3fe830e7b6b42afc49d8329b07a049a47d12e0ef9225f2fd220f19b2", - "sha256:61056a1fd3254f6f863de94c233b30dd33bc02f8c935b2000269705f1eeeffa4", - "sha256:67c8eb79ab33d0fbcb56842992298ddb56eb6505a72369c20f60bc1d2b6fb002", - "sha256:6e45bb4635b3c4e0a00ca9df75ef6295838c85c2ac44ad882410cb631ed1eeaa", - "sha256:7cb51096a6a8d400724104db8a7e4f2206041a1f23e58924aa3d8d96bcb48338", - "sha256:800a2b05cfb83654df80266692f7092eeefe2a314fa7901dcefab255934faeec", - "sha256:8df69e41f7e7015a90b94d1096ec3d8e0182e73449487306709ec27379fff761", - "sha256:917033016ecc23c8933205585a0ab73e20020fdf671b7cd1be788a5c4039840b", - "sha256:a12144d785518f6491ad334c75ccdc6ad52ea49230b4237f319dbb7cef26f464", - "sha256:a3866d68e2fc345162b1b9b83ef80686acfe5cec0d134337f3b03950a0a8bf56", - "sha256:a588a1cb7781da9d5e1c84affd98c32aff9c89771eac8eaa659d2760666f7139", - "sha256:a77b79852175064c822b047fee7cf5a1f434f06ad075cc9986aa1c19a0c53eb0", - "sha256:af83a554b3f077564229865c45af0791be008ac6469ef0098152139e6bd4b5b6", - "sha256:b801216c48c0886742abf286a9a6b117e248ca144d8ceec1f931ce2dd0c9cb40", - "sha256:bfb040b5dda1dff1e197d2ef71927bd6b8bfcb9793bc4dfe0bb6df1e691eaacb", - "sha256:c01678aee8ac0c1a461cbc38ad496f953f9efcb1fa19f5637cbeba7544792a53", - "sha256:c74eb1f73f788facece7979ce91594dc177e1a9b5d5e3e64697dd58299e5cb4d", - "sha256:c9a68a2f7bd091ccea54ad3be3e9d65eded813e6d79fdf4cc3604e26cdd6384f", - "sha256:d4dd3b381ff5a5907a3eb98f5f6d32c64d319a840278ceea1dcfcc65063856f3", - "sha256:e8e5ecbd4da4157889fce8ba49da74764dd86c891410bfd6b24969fa46edda51", - "sha256:eb2fc0ec241bf5e5ef56c8fbec4a2634d631e4c4f616a59b567947a0f35ad83c", - "sha256:edbe083c299835de7e02c8aa0885cb904a75087d35e7bab75ebe5ed336e8c3e2", - "sha256:ff64fd720def623bf64d8776f8d0deada1cc1bf1ec3c1f9d6f5bb5bd098d034f" + "sha256:0daad007b685db36d977f9de73f61f8da2a7104e20aca3effd30752fd56f73e1", + "sha256:108e5f1c1cd70ffce0b68739c75734437c919d2eaec8e85bffc2c8b4d2794305", + "sha256:19764605feea0df966445d46533729b645033f134baeb3ea26ad518c9fdf212c", + "sha256:1be97461c439a6af4fe1cf8bf6ca5936d3db252737d2f379cc6b2e394e12a458", + "sha256:25cd61e846aaab76d5791d006497134602a9e451e954833018161befc3b5b9ed", + "sha256:2a47bcc478741b71273b917232f521fd5704ab4b25d301669879e7273d3586cc", + "sha256:59af01efb011b0e8b686ba7758d59cf4a8263f9ad35911bfe3f416cee4f5c08c", + "sha256:5dcac11031a71348faaed1f403a0debd56bf5404232284cf8c761ff918886ebc", + "sha256:62a5ec91388984909bb5398ea49ee61b68ecb579123694bffa172c3b0a107079", + "sha256:645bd4ca6f543685d643dadf6a856cc382b654cc923460e3a10a49c1b3832aeb", + "sha256:653b29b0819605fe0898829c8ad6400a6ccde096146730c2da54eede9b7b8baa", + "sha256:69138068268127cd605e03438312d8f271135a33140e2742b417d027a0539427", + "sha256:6e186342cfcc3aafaad565cbd496060e5a614b441cacc3995ef0091115c1f6c5", + "sha256:76bd15bb65c14900d98835fcd10f59e5e0435077431d3a394b60b15864fddd64", + "sha256:7805830e0c56d88f4d491fa5ac640dfc894c5ec570d1ece6ed1546e9df2e98d6", + "sha256:7a710b79baddd65b806402e14766c721aee8fb83381769c27920f26476276c1e", + "sha256:7a7a8f33a1f1fb762ede6cc9cbab8f2a9ba13b196bfaf7bc6f0b39d2ba315a43", + "sha256:82ee7696ed8eb9a82c7037f32ba9b7c59e51dda6f105b39f043b6ef293989cb3", + "sha256:88afd7a3af7ddddd42c2deda43d53d3dfc016c11327d0915f90ca34ebda91499", + "sha256:8af1a451ff9e123d0d8bd5d5e60f8e3315c3a64f3cdd6bc853e26090e195cdc8", + "sha256:8ee606964553c1a0bc74057dd8782a37d1c2bc0f01b83193b6f8bb14523b877b", + "sha256:91852d4480a4537d169c29a9d104dda44094c78f1f5b67bca76c29a91042b623", + "sha256:9c682436c359b5ada67e882fec34689726a09c461efd75b6ea77b2403d5665b7", + "sha256:bc3ee1b4d97081260d92ae813a83de4d2653206967c4a0a017580f8b9548ddbc", + "sha256:bca649483d5ed251d06daf25957f802e44e6bb6df2e8f218ae71968ff8f8edc4", + "sha256:c39778fd0548d78917b61f03c1fa8bfda6cfcf98c767decf360945fe6f97461e", + "sha256:cbe71b6712429650e3883dc81286edb94c328ffcd24849accac0a4dbcc76958a", + "sha256:d00fe8596e1cc46b44bf3907354e9377aa030ec4cd04afbbf6e899fc1e2a7781", + "sha256:d3584623e68a5064a04748fb6d76117a21a7cb5eaba20608a41c7d0c61721794", + "sha256:e48217c7901edd95f9f097feaa0388da215ed14ce2ece803d3f300b4e694abea", + "sha256:f2e497413560e03421484189a6b65e33fe800d3bd75590e6d78d4dfdb7accf3b", + "sha256:ff5c9a67f8a4fba4aed887216e32cbc48f2a6fb2673bb10a99e43be463e15913" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==3.19.0" + "version": "==3.20.0" }, "pygments": { "hashes": [ @@ -1308,6 +1388,14 @@ "markers": "python_version >= '3.7'", "version": "==2.17.2" }, + "pyopenssl": { + "hashes": [ + "sha256:6aa33039a93fffa4563e655b61d11364d01264be8ccb49906101e02a334530bf", + "sha256:ba07553fb6fd6a7a2259adb9b84e12302a9a8a75c44046e8bb5d3e5ee887e3c3" + ], + "markers": "python_version >= '3.7'", + "version": "==24.0.0" + }, "pyparsing": { "hashes": [ "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb", @@ -1324,6 +1412,15 @@ ], "version": "==1.7.1" }, + "pysubs2": { + "hashes": [ + "sha256:0261611e71735ff7763972c519c72593c8063efcb9039c54af65f31b81cec116", + "sha256:1f96d9dfb5f859a54a00e04621beb20ff21ea1d788821b2f4935c5c0ef8dc68e" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==1.6.1" + }, "python-dateutil": { "hashes": [ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", @@ -1334,12 +1431,12 @@ }, "python-slugify": { "hashes": [ - "sha256:70ca6ea68fe63ecc8fa4fcf00ae651fc8a5d02d93dcd12ae6d4fc7ca46c4d395", - "sha256:ce0d46ddb668b3be82f4ed5e503dbc33dd815d83e2eb6824211310d3fb172a27" + "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8", + "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==8.0.1" + "version": "==8.0.4" }, "python-twitter-v2": { "hashes": [ @@ -1352,10 +1449,10 @@ }, "pytz": { "hashes": [ - "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b", - "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7" + "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812", + "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319" ], - "version": "==2023.3.post1" + "version": "==2024.1" }, "pyyaml": { "hashes": [ @@ -1388,6 +1485,7 @@ "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4", "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba", "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8", + "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef", "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5", "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd", "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3", @@ -1416,97 +1514,102 @@ }, "regex": { "hashes": [ - "sha256:00ba3c9818e33f1fa974693fb55d24cdc8ebafcb2e4207680669d8f8d7cca79a", - "sha256:00e871d83a45eee2f8688d7e6849609c2ca2a04a6d48fba3dff4deef35d14f07", - "sha256:06e9abc0e4c9ab4779c74ad99c3fc10d3967d03114449acc2c2762ad4472b8ca", - "sha256:0b9ac09853b2a3e0d0082104036579809679e7715671cfbf89d83c1cb2a30f58", - "sha256:0d47840dc05e0ba04fe2e26f15126de7c755496d5a8aae4a08bda4dd8d646c54", - "sha256:0f649fa32fe734c4abdfd4edbb8381c74abf5f34bc0b3271ce687b23729299ed", - "sha256:107ac60d1bfdc3edb53be75e2a52aff7481b92817cfdddd9b4519ccf0e54a6ff", - "sha256:11175910f62b2b8c055f2b089e0fedd694fe2be3941b3e2633653bc51064c528", - "sha256:12bd4bc2c632742c7ce20db48e0d99afdc05e03f0b4c1af90542e05b809a03d9", - "sha256:16f8740eb6dbacc7113e3097b0a36065a02e37b47c936b551805d40340fb9971", - "sha256:1c0e8fae5b27caa34177bdfa5a960c46ff2f78ee2d45c6db15ae3f64ecadde14", - "sha256:2c54e23836650bdf2c18222c87f6f840d4943944146ca479858404fedeb9f9af", - "sha256:3367007ad1951fde612bf65b0dffc8fd681a4ab98ac86957d16491400d661302", - "sha256:36362386b813fa6c9146da6149a001b7bd063dabc4d49522a1f7aa65b725c7ec", - "sha256:39807cbcbe406efca2a233884e169d056c35aa7e9f343d4e78665246a332f597", - "sha256:39cdf8d141d6d44e8d5a12a8569d5a227f645c87df4f92179bd06e2e2705e76b", - "sha256:3b2c3502603fab52d7619b882c25a6850b766ebd1b18de3df23b2f939360e1bd", - "sha256:3ccf2716add72f80714b9a63899b67fa711b654be3fcdd34fa391d2d274ce767", - "sha256:3fef4f844d2290ee0ba57addcec17eec9e3df73f10a2748485dfd6a3a188cc0f", - "sha256:4023e2efc35a30e66e938de5aef42b520c20e7eda7bb5fb12c35e5d09a4c43f6", - "sha256:4a3ee019a9befe84fa3e917a2dd378807e423d013377a884c1970a3c2792d293", - "sha256:4a8bf76e3182797c6b1afa5b822d1d5802ff30284abe4599e1247be4fd6b03be", - "sha256:4a992f702c9be9c72fa46f01ca6e18d131906a7180950958f766c2aa294d4b41", - "sha256:4c34d4f73ea738223a094d8e0ffd6d2c1a1b4c175da34d6b0de3d8d69bee6bcc", - "sha256:4cd1bccf99d3ef1ab6ba835308ad85be040e6a11b0977ef7ea8c8005f01a3c29", - "sha256:4ef80829117a8061f974b2fda8ec799717242353bff55f8a29411794d635d964", - "sha256:58837f9d221744d4c92d2cf7201c6acd19623b50c643b56992cbd2b745485d3d", - "sha256:5a8f91c64f390ecee09ff793319f30a0f32492e99f5dc1c72bc361f23ccd0a9a", - "sha256:5addc9d0209a9afca5fc070f93b726bf7003bd63a427f65ef797a931782e7edc", - "sha256:6239d4e2e0b52c8bd38c51b760cd870069f0bdf99700a62cd509d7a031749a55", - "sha256:66e2fe786ef28da2b28e222c89502b2af984858091675044d93cb50e6f46d7af", - "sha256:69c0771ca5653c7d4b65203cbfc5e66db9375f1078689459fe196fe08b7b4930", - "sha256:6ac965a998e1388e6ff2e9781f499ad1eaa41e962a40d11c7823c9952c77123e", - "sha256:6c56c3d47da04f921b73ff9415fbaa939f684d47293f071aa9cbb13c94afc17d", - "sha256:6f85739e80d13644b981a88f529d79c5bdf646b460ba190bffcaf6d57b2a9863", - "sha256:706e7b739fdd17cb89e1fbf712d9dc21311fc2333f6d435eac2d4ee81985098c", - "sha256:741ba2f511cc9626b7561a440f87d658aabb3d6b744a86a3c025f866b4d19e7f", - "sha256:7434a61b158be563c1362d9071358f8ab91b8d928728cd2882af060481244c9e", - "sha256:76066d7ff61ba6bf3cb5efe2428fc82aac91802844c022d849a1f0f53820502d", - "sha256:7979b834ec7a33aafae34a90aad9f914c41fd6eaa8474e66953f3f6f7cbd4368", - "sha256:7eece6fbd3eae4a92d7c748ae825cbc1ee41a89bb1c3db05b5578ed3cfcfd7cb", - "sha256:7ef1e014eed78ab650bef9a6a9cbe50b052c0aebe553fb2881e0453717573f52", - "sha256:81dce2ddc9f6e8f543d94b05d56e70d03a0774d32f6cca53e978dc01e4fc75b8", - "sha256:82fcc1f1cc3ff1ab8a57ba619b149b907072e750815c5ba63e7aa2e1163384a4", - "sha256:8d1f21af4c1539051049796a0f50aa342f9a27cde57318f2fc41ed50b0dbc4ac", - "sha256:90a79bce019c442604662d17bf69df99090e24cdc6ad95b18b6725c2988a490e", - "sha256:9145f092b5d1977ec8c0ab46e7b3381b2fd069957b9862a43bd383e5c01d18c2", - "sha256:91dc1d531f80c862441d7b66c4505cd6ea9d312f01fb2f4654f40c6fdf5cc37a", - "sha256:979c24cbefaf2420c4e377ecd1f165ea08cc3d1fbb44bdc51bccbbf7c66a2cb4", - "sha256:994645a46c6a740ee8ce8df7911d4aee458d9b1bc5639bc968226763d07f00fa", - "sha256:9b98b7681a9437262947f41c7fac567c7e1f6eddd94b0483596d320092004533", - "sha256:9c6b4d23c04831e3ab61717a707a5d763b300213db49ca680edf8bf13ab5d91b", - "sha256:9c6d0ced3c06d0f183b73d3c5920727268d2201aa0fe6d55c60d68c792ff3588", - "sha256:9fd88f373cb71e6b59b7fa597e47e518282455c2734fd4306a05ca219a1991b0", - "sha256:a8f4e49fc3ce020f65411432183e6775f24e02dff617281094ba6ab079ef0915", - "sha256:a9e908ef5889cda4de038892b9accc36d33d72fb3e12c747e2799a0e806ec841", - "sha256:ad08a69728ff3c79866d729b095872afe1e0557251da4abb2c5faff15a91d19a", - "sha256:adbccd17dcaff65704c856bd29951c58a1bd4b2b0f8ad6b826dbd543fe740988", - "sha256:b0c7d2f698e83f15228ba41c135501cfe7d5740181d5903e250e47f617eb4292", - "sha256:b3ab05a182c7937fb374f7e946f04fb23a0c0699c0450e9fb02ef567412d2fa3", - "sha256:b6104f9a46bd8743e4f738afef69b153c4b8b592d35ae46db07fc28ae3d5fb7c", - "sha256:ba7cd6dc4d585ea544c1412019921570ebd8a597fabf475acc4528210d7c4a6f", - "sha256:bc72c231f5449d86d6c7d9cc7cd819b6eb30134bb770b8cfdc0765e48ef9c420", - "sha256:bce8814b076f0ce5766dc87d5a056b0e9437b8e0cd351b9a6c4e1134a7dfbda9", - "sha256:be5e22bbb67924dea15039c3282fa4cc6cdfbe0cbbd1c0515f9223186fc2ec5f", - "sha256:be6b7b8d42d3090b6c80793524fa66c57ad7ee3fe9722b258aec6d0672543fd0", - "sha256:bfe50b61bab1b1ec260fa7cd91106fa9fece57e6beba05630afe27c71259c59b", - "sha256:bff507ae210371d4b1fe316d03433ac099f184d570a1a611e541923f78f05037", - "sha256:c148bec483cc4b421562b4bcedb8e28a3b84fcc8f0aa4418e10898f3c2c0eb9b", - "sha256:c15ad0aee158a15e17e0495e1e18741573d04eb6da06d8b84af726cfc1ed02ee", - "sha256:c2169b2dcabf4e608416f7f9468737583ce5f0a6e8677c4efbf795ce81109d7c", - "sha256:c55853684fe08d4897c37dfc5faeff70607a5f1806c8be148f1695be4a63414b", - "sha256:c65a3b5330b54103e7d21cac3f6bf3900d46f6d50138d73343d9e5b2900b2353", - "sha256:c7964c2183c3e6cce3f497e3a9f49d182e969f2dc3aeeadfa18945ff7bdd7051", - "sha256:cc3f1c053b73f20c7ad88b0d1d23be7e7b3901229ce89f5000a8399746a6e039", - "sha256:ce615c92d90df8373d9e13acddd154152645c0dc060871abf6bd43809673d20a", - "sha256:d29338556a59423d9ff7b6eb0cb89ead2b0875e08fe522f3e068b955c3e7b59b", - "sha256:d8a993c0a0ffd5f2d3bda23d0cd75e7086736f8f8268de8a82fbc4bd0ac6791e", - "sha256:d9c727bbcf0065cbb20f39d2b4f932f8fa1631c3e01fcedc979bd4f51fe051c5", - "sha256:dac37cf08fcf2094159922edc7a2784cfcc5c70f8354469f79ed085f0328ebdf", - "sha256:dd829712de97753367153ed84f2de752b86cd1f7a88b55a3a775eb52eafe8a94", - "sha256:e54ddd0bb8fb626aa1f9ba7b36629564544954fff9669b15da3610c22b9a0991", - "sha256:e77c90ab5997e85901da85131fd36acd0ed2221368199b65f0d11bca44549711", - "sha256:ebedc192abbc7fd13c5ee800e83a6df252bec691eb2c4bedc9f8b2e2903f5e2a", - "sha256:ef71561f82a89af6cfcbee47f0fabfdb6e63788a9258e913955d89fdd96902ab", - "sha256:f0a47efb1dbef13af9c9a54a94a0b814902e547b7f21acb29434504d18f36e3a", - "sha256:f4f2ca6df64cbdd27f27b34f35adb640b5d2d77264228554e68deda54456eb11", - "sha256:fb02e4257376ae25c6dd95a5aec377f9b18c09be6ebdefa7ad209b9137b73d48" + "sha256:0694219a1d54336fd0445ea382d49d36882415c0134ee1e8332afd1529f0baa5", + "sha256:086dd15e9435b393ae06f96ab69ab2d333f5d65cbe65ca5a3ef0ec9564dfe770", + "sha256:094ba386bb5c01e54e14434d4caabf6583334090865b23ef58e0424a6286d3dc", + "sha256:09da66917262d9481c719599116c7dc0c321ffcec4b1f510c4f8a066f8768105", + "sha256:0ecf44ddf9171cd7566ef1768047f6e66975788258b1c6c6ca78098b95cf9a3d", + "sha256:0fda75704357805eb953a3ee15a2b240694a9a514548cd49b3c5124b4e2ad01b", + "sha256:11a963f8e25ab5c61348d090bf1b07f1953929c13bd2309a0662e9ff680763c9", + "sha256:150c39f5b964e4d7dba46a7962a088fbc91f06e606f023ce57bb347a3b2d4630", + "sha256:1b9d811f72210fa9306aeb88385b8f8bcef0dfbf3873410413c00aa94c56c2b6", + "sha256:1e0eabac536b4cc7f57a5f3d095bfa557860ab912f25965e08fe1545e2ed8b4c", + "sha256:22a86d9fff2009302c440b9d799ef2fe322416d2d58fc124b926aa89365ec482", + "sha256:22f3470f7524b6da61e2020672df2f3063676aff444db1daa283c2ea4ed259d6", + "sha256:263ef5cc10979837f243950637fffb06e8daed7f1ac1e39d5910fd29929e489a", + "sha256:283fc8eed679758de38fe493b7d7d84a198b558942b03f017b1f94dda8efae80", + "sha256:29171aa128da69afdf4bde412d5bedc335f2ca8fcfe4489038577d05f16181e5", + "sha256:298dc6354d414bc921581be85695d18912bea163a8b23cac9a2562bbcd5088b1", + "sha256:2aae8101919e8aa05ecfe6322b278f41ce2994c4a430303c4cd163fef746e04f", + "sha256:2f4e475a80ecbd15896a976aa0b386c5525d0ed34d5c600b6d3ebac0a67c7ddf", + "sha256:34e4af5b27232f68042aa40a91c3b9bb4da0eeb31b7632e0091afc4310afe6cb", + "sha256:37f8e93a81fc5e5bd8db7e10e62dc64261bcd88f8d7e6640aaebe9bc180d9ce2", + "sha256:3a17d3ede18f9cedcbe23d2daa8a2cd6f59fe2bf082c567e43083bba3fb00347", + "sha256:3b1de218d5375cd6ac4b5493e0b9f3df2be331e86520f23382f216c137913d20", + "sha256:43f7cd5754d02a56ae4ebb91b33461dc67be8e3e0153f593c509e21d219c5060", + "sha256:4558410b7a5607a645e9804a3e9dd509af12fb72b9825b13791a37cd417d73a5", + "sha256:4719bb05094d7d8563a450cf8738d2e1061420f79cfcc1fa7f0a44744c4d8f73", + "sha256:4bfc2b16e3ba8850e0e262467275dd4d62f0d045e0e9eda2bc65078c0110a11f", + "sha256:518440c991f514331f4850a63560321f833979d145d7d81186dbe2f19e27ae3d", + "sha256:51f4b32f793812714fd5307222a7f77e739b9bc566dc94a18126aba3b92b98a3", + "sha256:531ac6cf22b53e0696f8e1d56ce2396311254eb806111ddd3922c9d937151dae", + "sha256:5cd05d0f57846d8ba4b71d9c00f6f37d6b97d5e5ef8b3c3840426a475c8f70f4", + "sha256:5dd58946bce44b53b06d94aa95560d0b243eb2fe64227cba50017a8d8b3cd3e2", + "sha256:60080bb3d8617d96f0fb7e19796384cc2467447ef1c491694850ebd3670bc457", + "sha256:636ba0a77de609d6510235b7f0e77ec494d2657108f777e8765efc060094c98c", + "sha256:67d3ccfc590e5e7197750fcb3a2915b416a53e2de847a728cfa60141054123d4", + "sha256:68191f80a9bad283432385961d9efe09d783bcd36ed35a60fb1ff3f1ec2efe87", + "sha256:7502534e55c7c36c0978c91ba6f61703faf7ce733715ca48f499d3dbbd7657e0", + "sha256:7aa47c2e9ea33a4a2a05f40fcd3ea36d73853a2aae7b4feab6fc85f8bf2c9704", + "sha256:7d2af3f6b8419661a0c421584cfe8aaec1c0e435ce7e47ee2a97e344b98f794f", + "sha256:7e316026cc1095f2a3e8cc012822c99f413b702eaa2ca5408a513609488cb62f", + "sha256:88ad44e220e22b63b0f8f81f007e8abbb92874d8ced66f32571ef8beb0643b2b", + "sha256:88d1f7bef20c721359d8675f7d9f8e414ec5003d8f642fdfd8087777ff7f94b5", + "sha256:89723d2112697feaa320c9d351e5f5e7b841e83f8b143dba8e2d2b5f04e10923", + "sha256:8a0ccf52bb37d1a700375a6b395bff5dd15c50acb745f7db30415bae3c2b0715", + "sha256:8c2c19dae8a3eb0ea45a8448356ed561be843b13cbc34b840922ddf565498c1c", + "sha256:905466ad1702ed4acfd67a902af50b8db1feeb9781436372261808df7a2a7bca", + "sha256:9852b76ab558e45b20bf1893b59af64a28bd3820b0c2efc80e0a70a4a3ea51c1", + "sha256:98a2636994f943b871786c9e82bfe7883ecdaba2ef5df54e1450fa9869d1f756", + "sha256:9aa1a67bbf0f957bbe096375887b2505f5d8ae16bf04488e8b0f334c36e31360", + "sha256:9eda5f7a50141291beda3edd00abc2d4a5b16c29c92daf8d5bd76934150f3edc", + "sha256:a6d1047952c0b8104a1d371f88f4ab62e6275567d4458c1e26e9627ad489b445", + "sha256:a9b6d73353f777630626f403b0652055ebfe8ff142a44ec2cf18ae470395766e", + "sha256:a9cc99d6946d750eb75827cb53c4371b8b0fe89c733a94b1573c9dd16ea6c9e4", + "sha256:ad83e7545b4ab69216cef4cc47e344d19622e28aabec61574b20257c65466d6a", + "sha256:b014333bd0217ad3d54c143de9d4b9a3ca1c5a29a6d0d554952ea071cff0f1f8", + "sha256:b43523d7bc2abd757119dbfb38af91b5735eea45537ec6ec3a5ec3f9562a1c53", + "sha256:b521dcecebc5b978b447f0f69b5b7f3840eac454862270406a39837ffae4e697", + "sha256:b77e27b79448e34c2c51c09836033056a0547aa360c45eeeb67803da7b0eedaf", + "sha256:b7a635871143661feccce3979e1727c4e094f2bdfd3ec4b90dfd4f16f571a87a", + "sha256:b7fca9205b59c1a3d5031f7e64ed627a1074730a51c2a80e97653e3e9fa0d415", + "sha256:ba1b30765a55acf15dce3f364e4928b80858fa8f979ad41f862358939bdd1f2f", + "sha256:ba99d8077424501b9616b43a2d208095746fb1284fc5ba490139651f971d39d9", + "sha256:c25a8ad70e716f96e13a637802813f65d8a6760ef48672aa3502f4c24ea8b400", + "sha256:c3c4a78615b7762740531c27cf46e2f388d8d727d0c0c739e72048beb26c8a9d", + "sha256:c40281f7d70baf6e0db0c2f7472b31609f5bc2748fe7275ea65a0b4601d9b392", + "sha256:c7ad32824b7f02bb3c9f80306d405a1d9b7bb89362d68b3c5a9be53836caebdb", + "sha256:cb3fe77aec8f1995611f966d0c656fdce398317f850d0e6e7aebdfe61f40e1cd", + "sha256:cc038b2d8b1470364b1888a98fd22d616fba2b6309c5b5f181ad4483e0017861", + "sha256:cc37b9aeebab425f11f27e5e9e6cf580be7206c6582a64467a14dda211abc232", + "sha256:cc6bb9aa69aacf0f6032c307da718f61a40cf970849e471254e0e91c56ffca95", + "sha256:d126361607b33c4eb7b36debc173bf25d7805847346dd4d99b5499e1fef52bc7", + "sha256:d15b274f9e15b1a0b7a45d2ac86d1f634d983ca40d6b886721626c47a400bf39", + "sha256:d166eafc19f4718df38887b2bbe1467a4f74a9830e8605089ea7a30dd4da8887", + "sha256:d498eea3f581fbe1b34b59c697512a8baef88212f92e4c7830fcc1499f5b45a5", + "sha256:d6f7e255e5fa94642a0724e35406e6cb7001c09d476ab5fce002f652b36d0c39", + "sha256:d78bd484930c1da2b9679290a41cdb25cc127d783768a0369d6b449e72f88beb", + "sha256:d865984b3f71f6d0af64d0d88f5733521698f6c16f445bb09ce746c92c97c586", + "sha256:d902a43085a308cef32c0d3aea962524b725403fd9373dea18110904003bac97", + "sha256:d94a1db462d5690ebf6ae86d11c5e420042b9898af5dcf278bd97d6bda065423", + "sha256:da695d75ac97cb1cd725adac136d25ca687da4536154cdc2815f576e4da11c69", + "sha256:db2a0b1857f18b11e3b0e54ddfefc96af46b0896fb678c85f63fb8c37518b3e7", + "sha256:df26481f0c7a3f8739fecb3e81bc9da3fcfae34d6c094563b9d4670b047312e1", + "sha256:e14b73607d6231f3cc4622809c196b540a6a44e903bcfad940779c80dffa7be7", + "sha256:e2610e9406d3b0073636a3a2e80db05a02f0c3169b5632022b4e81c0364bcda5", + "sha256:e692296c4cc2873967771345a876bcfc1c547e8dd695c6b89342488b0ea55cd8", + "sha256:e693e233ac92ba83a87024e1d32b5f9ab15ca55ddd916d878146f4e3406b5c91", + "sha256:e81469f7d01efed9b53740aedd26085f20d49da65f9c1f41e822a33992cb1590", + "sha256:e8c7e08bb566de4faaf11984af13f6bcf6a08f327b13631d41d62592681d24fe", + "sha256:ed19b3a05ae0c97dd8f75a5d8f21f7723a8c33bbc555da6bbe1f96c470139d3c", + "sha256:efb2d82f33b2212898f1659fb1c2e9ac30493ac41e4d53123da374c3b5541e64", + "sha256:f44dd4d68697559d007462b0a3a1d9acd61d97072b71f6d1968daef26bc744bd", + "sha256:f72cbae7f6b01591f90814250e636065850c5926751af02bb48da94dfced7baa", + "sha256:f7bc09bc9c29ebead055bcba136a67378f03d66bf359e87d0f7c759d6d4ffa31", + "sha256:ff100b203092af77d1a5a7abe085b3506b7eaaf9abf65b73b7d6905b6cb76988" ], "markers": "python_version >= '3.7'", - "version": "==2023.10.3" + "version": "==2023.12.25" }, "requests": { "extras": [ @@ -1535,6 +1638,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.0" }, + "retrying": { + "hashes": [ + "sha256:345da8c5765bd982b1d1915deb9102fd3d1f7ad16bd84a9700b85f64d24e8f3e", + "sha256:8cc4d43cb8e1125e0ff3344e9de678fefd85db3b750b81b2240dc0183af37b35" + ], + "index": "pypi", + "version": "==1.3.4" + }, "rich": { "hashes": [ "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa", @@ -1553,20 +1664,20 @@ }, "s3transfer": { "hashes": [ - "sha256:01d4d2c35a016db8cb14f9a4d5e84c1f8c96e7ffc211422555eed45c11fa7eb1", - "sha256:9e1b186ec8bb5907a1e82b51237091889a9973a2bb799a924bcd9f301ff79d3d" + "sha256:3cdb40f5cfa6966e812209d0994f2a4709b561c88e90cf00c2696d2df4e56b2e", + "sha256:d0c8bbf672d5eebbe4e57945e23b972d963f07d82f661cabf678a5c88831595b" ], "markers": "python_version >= '3.8'", - "version": "==0.9.0" + "version": "==0.10.0" }, "selenium": { "hashes": [ - "sha256:aec71f4e6ed6cb3ec25c9c1b5ed56ae31b6da0a7f17474c7566d303f84e6219f", - "sha256:b2e987a445306151f7be0e6dfe2aa72a479c2ac6a91b9d5ef2d6dd4e49ad0435" + "sha256:a11f67afa8bfac6b77e148c987b33f6b14eb1cae4d352722a75de1f26e3f0ae2", + "sha256:b24a3cdd2d47c29832e81345bfcde0c12bb608738013e53c781b211b418df241" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==4.16.0" + "version": "==4.18.1" }, "six": { "hashes": [ @@ -1608,13 +1719,20 @@ "markers": "python_version >= '3.8'", "version": "==2.5" }, + "strenum": { + "hashes": [ + "sha256:878fb5ab705442070e4dd1929bb5e2249511c0bcf2b0eeacf3bcd80875c82eff", + "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659" + ], + "version": "==0.4.15" + }, "telethon": { "hashes": [ - "sha256:377107d77fd95f8c2bd7fce77f9d78da84c1ff58023f59e8d06035a4548bd36b" + "sha256:55290809a30081fa0bb5052abb7547cbb25d7fbca94f32f13c147504d521804f" ], "index": "pypi", "markers": "python_version >= '3.5'", - "version": "==1.33.1" + "version": "==1.34.0" }, "text-unidecode": { "hashes": [ @@ -1632,20 +1750,20 @@ }, "tqdm": { "hashes": [ - "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386", - "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7" + "sha256:1ee4f8a893eb9bef51c6e35730cebf234d5d0b6bd112b0271e10ed7c24a02bd9", + "sha256:6cd52cdf0fef0e0f543299cfc96fec90d7b8a7e88745f411ec33eb44d5ed3531" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==4.66.1" + "version": "==4.66.2" }, "trio": { "hashes": [ - "sha256:5a0b566fa5d50cf231cfd6b08f3b03aa4179ff004b8f3144059587039e2b26d3", - "sha256:da1d35b9a2b17eb32cae2e763b16551f9aa6703634735024e32f325c9285069e" + "sha256:c3bd3a4e3e3025cd9a2241eae75637c43fe0b9e88b4c97b9161a55b9e54cd72c", + "sha256:ffa09a74a6bf81b84f8613909fb0beaee84757450183a7a2e0b47b455c0cac5d" ], "markers": "python_version >= '3.8'", - "version": "==0.23.2" + "version": "==0.24.0" }, "trio-websocket": { "hashes": [ @@ -1655,6 +1773,14 @@ "markers": "python_version >= '3.7'", "version": "==0.11.1" }, + "tsp-client": { + "hashes": [ + "sha256:82c02fc2383e94029d34feeeb438455283afd81805b1d36a92c3a801e0f29a0f", + "sha256:941359c08273a7ecf430c71a66b97c8d0beb69152df31ddf9ec2ad86d1144437" + ], + "index": "pypi", + "version": "==0.1.4" + }, "typing-extensions": { "hashes": [ "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783", @@ -1687,11 +1813,14 @@ "version": "==4.1.1" }, "urllib3": { + "extras": [ + "socks" + ], "hashes": [ "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "markers": "python_version >= '3.7'", "version": "==2.0.7" }, "vk-api": { @@ -1703,12 +1832,12 @@ }, "vk-url-scraper": { "hashes": [ - "sha256:b81caa8738b0154aca25ab33012e8e4d8dc11cdc9fd371968afbbadd57e2e1ab", - "sha256:d12aec2828e8e47c766991207fd809e402ab832ea449129b94ac7eadb0d1a27c" + "sha256:133d252ee94ceb1ee9515fb448d410ba471cbccc19e303b548076cd44cc81f30", + "sha256:c1c001b66b80343a991628080398d8a923e8753183b952f99f40ecafe1087070" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==0.3.26" + "version": "==0.3.27" }, "warcio": { "hashes": [ @@ -1910,12 +2039,12 @@ }, "yt-dlp": { "hashes": [ - "sha256:0322ba85aa4afdb75f8641ed550e5958964daff034aeb477abb15031fd9a51ed", - "sha256:f0ccdaf12e08b15902601a4671c7ab12906d7b11de3ae75fa6506811c24ec5da" + "sha256:a11862e57721b0a0f0883dfeb5a4d79ba213a2d4c45e1880e9fd70f8e6570c38", + "sha256:c00d9a71d64472ad441bcaa1ec0c3797d6e60c9f934f270096a96fe51657e7b3" ], "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2023.11.16" + "markers": "python_version >= '3.8'", + "version": "==2023.12.30" } }, "develop": { diff --git a/README.md b/README.md index 7972184..62baa31 100644 --- a/README.md +++ b/README.md @@ -233,12 +233,12 @@ working with docker locally: * to use local archive, also create a volume `-v` for it by adding `-v $PWD/local_archive:/app/local_archive` -release to docker hub +manual release to docker hub * `docker image tag auto-archiver bellingcat/auto-archiver:latest` * `docker push bellingcat/auto-archiver` #### RELEASE * update version in [version.py](src/auto_archiver/version.py) -* run `bash ./scripts/release.sh` and confirm -* package is automatically updated in pypi -* docker image is automatically pushed to dockerhup +* go to github releases > new release > use `vx.y.z` for matching version notation + * package is automatically updated in pypi + * docker image is automatically pushed to dockerhup diff --git a/scripts/release.sh b/scripts/release.sh deleted file mode 100755 index c063bfa..0000000 --- a/scripts/release.sh +++ /dev/null @@ -1,19 +0,0 @@ - -#!/bin/bash - -set -e - -TAG=$(python -c 'from src.auto_archiver.version import __version__; print("v" + __version__)') - -read -p "Creating new release for $TAG. Do you want to continue? [Y/n] " prompt - -if [[ $prompt == "y" || $prompt == "Y" || $prompt == "yes" || $prompt == "Yes" ]]; then - # git add -A - # git commit -m "Bump version to $TAG for release" || true && git push - echo "Creating new git tag $TAG" - git tag "$TAG" -m "$TAG" - git push --tags -else - echo "Cancelled" - exit 1 -fi \ No newline at end of file diff --git a/src/auto_archiver/archivers/__init__.py b/src/auto_archiver/archivers/__init__.py index f9cbb55..ac92fde 100644 --- a/src/auto_archiver/archivers/__init__.py +++ b/src/auto_archiver/archivers/__init__.py @@ -7,4 +7,5 @@ from .instagram_tbot_archiver import InstagramTbotArchiver from .tiktok_archiver import TiktokArchiver from .telegram_archiver import TelegramArchiver from .vk_archiver import VkArchiver -from .youtubedl_archiver import YoutubeDLArchiver \ No newline at end of file +from .youtubedl_archiver import YoutubeDLArchiver +from .instagram_api_archiver import InstagramAPIArchiver \ No newline at end of file diff --git a/src/auto_archiver/archivers/archiver.py b/src/auto_archiver/archivers/archiver.py index 2928ecf..c44ab0a 100644 --- a/src/auto_archiver/archivers/archiver.py +++ b/src/auto_archiver/archivers/archiver.py @@ -3,6 +3,8 @@ from abc import abstractmethod from dataclasses import dataclass import os import mimetypes, requests +from loguru import logger +from retrying import retry from ..core import Metadata, Step, ArchivingContext @@ -23,6 +25,10 @@ class Archiver(Step): # used when archivers need to login or do other one-time setup pass + def cleanup(self) -> None: + # called when archivers are done, or upon errors, cleanup any resources + pass + def sanitize_url(self, url: str) -> str: # used to clean unnecessary URL parameters OR unfurl redirect links return url @@ -37,16 +43,17 @@ class Archiver(Step): return mime.split("/")[0] return "" - def download_from_url(self, url: str, to_filename: str = None, item: Metadata = None) -> str: + @retry(wait_random_min=500, wait_random_max=3500, stop_max_attempt_number=5) + def download_from_url(self, url: str, to_filename: str = None, verbose=True) -> str: """ - downloads a URL to provided filename, or inferred from URL, returns local filename, if item is present will use its tmp_dir + downloads a URL to provided filename, or inferred from URL, returns local filename """ if not to_filename: to_filename = url.split('/')[-1].split('?')[0] if len(to_filename) > 64: to_filename = to_filename[-64:] - if item: - to_filename = os.path.join(ArchivingContext.get_tmp_dir(), to_filename) + to_filename = os.path.join(ArchivingContext.get_tmp_dir(), to_filename) + if verbose: logger.debug(f"downloading {url[0:50]=} {to_filename=}") headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36' } diff --git a/src/auto_archiver/archivers/instagram_api_archiver.py b/src/auto_archiver/archivers/instagram_api_archiver.py new file mode 100644 index 0000000..edde209 --- /dev/null +++ b/src/auto_archiver/archivers/instagram_api_archiver.py @@ -0,0 +1,272 @@ +import re, requests +from datetime import datetime +from loguru import logger +from retrying import retry +from tqdm import tqdm + +from . import Archiver +from ..core import Metadata +from ..core import Media + +class InstagramAPIArchiver(Archiver): + """ + Uses an https://github.com/subzeroid/instagrapi API deployment to fetch instagram posts data + + # TODO: improvement collect aggregates of locations[0].location and mentions for all posts + """ + name = "instagram_api_archiver" + + global_pattern = re.compile(r"(?:(?:http|https):\/\/)?(?:www.)?(?:instagram.com)\/(stories(?:\/highlights)?|p|reel)?\/?([^\/\?]*)\/?(\d+)?") + + def __init__(self, config: dict) -> None: + super().__init__(config) + self.assert_valid_string("access_token") + self.assert_valid_string("api_endpoint") + if self.api_endpoint[-1] == "/": self.api_endpoint = self.api_endpoint[:-1] + + self.full_profile = bool(self.full_profile) + self.minimize_json_output = bool(self.minimize_json_output) + + @staticmethod + def configs() -> dict: + return { + "access_token": {"default": None, "help": "a valid instagrapi-api token"}, + "api_endpoint": {"default": None, "help": "API endpoint to use"}, + "full_profile": {"default": False, "help": "if true, will download all posts, tagged posts, stories, and highlights for a profile, if false, will only download the profile pic and information."}, + "minimize_json_output": {"default": True, "help": "if true, will remove empty values from the json output"}, + } + + def download(self, item: Metadata) -> Metadata: + url = item.get_url() + + url.replace("instagr.com", "instagram.com").replace("instagr.am", "instagram.com") + insta_matches = self.global_pattern.findall(url) + logger.info(f"{insta_matches=}") + if not len(insta_matches) or len(insta_matches[0])!=3: return + if len(insta_matches) > 1: + logger.warning(f"Multiple instagram matches found in {url=}, using the first one") + return + g1, g2, g3 = insta_matches[0][0], insta_matches[0][1], insta_matches[0][2] + if g1 == "": return self.download_profile(item, g2) + elif g1 == "p": return self.download_post(item, g2, context="post") + elif g1 == "reel": return self.download_post(item, g2, context="reel") + elif g1 == "stories/highlights": return self.download_highlights(item, g2) + elif g1 == "stories": + if len(g3): return self.download_post(item, id=g3, context="story") + return self.download_stories(item, g2) + else: + logger.warning(f"Unknown instagram regex group match {g1=} found in {url=}") + return + + @retry(wait_random_min=1000, wait_random_max=3000, stop_max_attempt_number=5) + def call_api(self, path: str, params: dict) -> dict: + headers = { + "accept": "application/json", + "x-access-key": self.access_token + } + logger.debug(f"calling {self.api_endpoint}/{path} with {params=}") + return requests.get(f"{self.api_endpoint}/{path}", headers=headers, params=params).json() + + def cleanup_dict(self, d: dict | list) -> dict: + # repeats 3 times to remove nested empty values + if not self.minimize_json_output: return d + if type(d) == list: return [self.cleanup_dict(v) for v in d] + if type(d) != dict: return d + return { + k: self.cleanup_dict(v) if type(v) in [dict, list] else v + for k, v in d.items() + if v not in [0.0, 0, [], {}, "", None, "null"] and + k not in ["x", "y", "width", "height"] + } + + def download_profile(self, result: Metadata, username: str) -> Metadata: + # download basic profile info + url = result.get_url() + user = self.call_api("v2/user/by/username", {"username": username}).get("user") + assert user, f"User {username} not found" + user = self.cleanup_dict(user) + + result.set_title(user.get("full_name", username)).set("data", user) + if pic_url := user.get("profile_pic_url_hd", user.get("profile_pic_url")): + filename = self.download_from_url(pic_url) + result.add_media(Media(filename=filename), id=f"profile_picture") + + if self.full_profile: + user_id = user.get("pk") + # download all posts + self.download_all_posts(result, user_id) + + # download all stories + try: + stories = self._download_stories_reusable(result, username) + result.set("#stories", len(stories)) + except Exception as e: + result.append("errors", f"Error downloading stories for {username}") + logger.error(f"Error downloading stories for {username}: {e}") + + # download all highlights + try: + count_highlights = 0 + highlights = self.call_api(f"v1/user/highlights", {"user_id": user_id}) + for h in highlights: + try: + h_info = self._download_highlights_reusable(result, h.get("pk")) + count_highlights += len(h_info.get("items", [])) + except Exception as e: + result.append("errors", f"Error downloading highlight id{h.get('pk')} for {username}") + logger.error(f"Error downloading highlight id{h.get('pk')} for {username}: {e}") + result.set("#highlights", count_highlights) + except Exception as e: + result.append("errors", f"Error downloading highlights for {username}") + logger.error(f"Error downloading highlights for {username}: {e}") + + result.set_url(url) # reset as scrape_item modifies it + return result.success("insta profile") + + def download_post(self, result: Metadata, code: str = None, id: str = None, context: str = None) -> Metadata: + if id: + post = self.call_api(f"v1/media/by/id", {"id": id}) + else: + post = self.call_api(f"v1/media/by/code", {"code": code}) + assert post, f"Post {id or code} not found" + + if caption_text := post.get("caption_text"): + result.set_title(caption_text) + + post = self.scrape_item(result, post, context) + + if post.get("taken_at"): result.set_timestamp(post.get("taken_at")) + return result.success(f"insta {context or 'post'}") + + def download_highlights(self, result: Metadata, id: str) -> Metadata: + h_info = self._download_highlights_reusable(result, id) + items = len(h_info.get("items", [])) + del h_info["items"] + result.set_title(h_info.get("title")).set("data", h_info).set("#reels", items) + return result.success("insta highlights") + + def _download_highlights_reusable(self, result: Metadata, id: str) ->dict: + full_h = self.call_api(f"v2/highlight/by/id", {"id": id}) + h_info = full_h.get("response", {}).get("reels", {}).get(f"highlight:{id}") + assert h_info, f"Highlight {id} not found: {full_h=}" + + if cover_media := h_info.get("cover_media", {}).get("cropped_image_version", {}).get("url"): + filename = self.download_from_url(cover_media) + result.add_media(Media(filename=filename), id=f"cover_media highlight {id}") + + items = h_info.get("items", [])[::-1] # newest to oldest + for h in tqdm(items, desc="downloading highlights", unit="highlight"): + try: self.scrape_item(result, h, "highlight") + except Exception as e: + result.append("errors", f"Error downloading highlight {h.get('id')}") + logger.error(f"Error downloading highlight, skipping {h.get('id')}: {e}") + + return h_info + + def download_stories(self, result: Metadata, username: str) -> Metadata: + now = datetime.now().strftime("%Y-%m-%d_%H-%M") + stories = self._download_stories_reusable(result, username) + result.set_title(f"stories {username} at {now}").set("#stories", len(stories)) + return result.success(f"insta stories {now}") + + def _download_stories_reusable(self, result: Metadata, username: str) -> list[dict]: + stories = self.call_api(f"v1/user/stories/by/username", {"username": username}) + assert stories, f"Stories for {username} not found" + stories = stories[::-1] # newest to oldest + + for s in tqdm(stories, desc="downloading stories", unit="story"): + try: self.scrape_item(result, s, "story") + except Exception as e: + result.append("errors", f"Error downloading story {s.get('id')}") + logger.error(f"Error downloading story, skipping {s.get('id')}: {e}") + return stories + + def download_all_posts(self, result: Metadata, user_id: str): + end_cursor = None + pbar = tqdm(desc="downloading posts") + + post_count = 0 + while end_cursor != "": + posts = self.call_api(f"v1/user/medias/chunk", {"user_id": user_id, "end_cursor": end_cursor}) + if not len(posts): break + posts, end_cursor = posts[0], posts[1] + logger.info(f"parsing {len(posts)} posts, next {end_cursor=}") + + for p in posts: + try: self.scrape_item(result, p, "post") + except Exception as e: + result.append("errors", f"Error downloading post {p.get('id')}") + logger.error(f"Error downloading post, skipping {p.get('id')}: {e}") + pbar.update(1) + post_count+=1 + result.set("#posts", post_count) + + +### reusable parsing utils below + + def scrape_item(self, result:Metadata, item:dict, context:str=None) -> dict: + """ + receives a Metadata and an API dict response + fetches the media and adds it to the Metadata + cleans and returns the API dict + context can be used to give specific id prefixes to media + """ + if "clips_metadata" in item: + if reusable_text := item.get("clips_metadata", {}).get("reusable_text_attribute_string"): + item["clips_metadata_text"] = reusable_text + if self.minimize_json_output: + del item["clips_metadata"] + + if code := item.get("code"): + result.set("url", f"https://www.instagram.com/p/{code}/") + + resources = item.get("resources", []) + item, media, media_id = self.scrape_media(item, context) + # if resources are present take the main media from the first resource + if not media and len(resources): + _, media, media_id = self.scrape_media(resources[0], context) + resources = resources[1:] + + assert media, f"Image/video not found in {item=}" + + # posts with multiple items contain a resources list + resources_metadata = Metadata() + for r in resources: + self.scrape_item(resources_metadata, r) + if not resources_metadata.is_empty(): + media.set("other media", resources_metadata.media) + + result.add_media(media, id=media_id) + return item + + def scrape_media(self, item: dict, context:str) -> tuple[dict, Media, str]: + # remove unnecessary info + if self.minimize_json_output: + for k in ["image_versions", "video_versions", "video_dash_manifest"]: + if k in item: del item[k] + item = self.cleanup_dict(item) + + image_media = None + if image_url := item.get("thumbnail_url"): + filename = self.download_from_url(image_url, verbose=False) + image_media = Media(filename=filename) + + # retrieve video info + best_id = item.get('id', item.get('pk')) + taken_at = item.get("taken_at") + code = item.get("code") + if video_url := item.get("video_url"): + filename = self.download_from_url(video_url, verbose=False) + video_media = Media(filename=filename) + if taken_at: video_media.set("date", taken_at) + if code: video_media.set("url", f"https://www.instagram.com/p/{code}") + video_media.set("preview", [image_media]) + video_media.set("data", [item]) + return item, video_media, f"{context or 'video'} {best_id}" + elif image_media: + if taken_at: image_media.set("date", taken_at) + if code: image_media.set("url", f"https://www.instagram.com/p/{code}") + image_media.set("data", [item]) + return item, image_media, f"{context or 'image'} {best_id}" + + return item, None, None \ No newline at end of file diff --git a/src/auto_archiver/archivers/instagram_tbot_archiver.py b/src/auto_archiver/archivers/instagram_tbot_archiver.py index ee41247..580b8ac 100644 --- a/src/auto_archiver/archivers/instagram_tbot_archiver.py +++ b/src/auto_archiver/archivers/instagram_tbot_archiver.py @@ -1,10 +1,12 @@ +import shutil from telethon.sync import TelegramClient from loguru import logger import time, os from sqlite3 import OperationalError from . import Archiver from ..core import Metadata, Media, ArchivingContext +from ..utils import random_str class InstagramTbotArchiver(Archiver): @@ -20,10 +22,6 @@ class InstagramTbotArchiver(Archiver): self.assert_valid_string("api_id") self.assert_valid_string("api_hash") self.timeout = int(self.timeout) - try: - self.client = TelegramClient(self.session_file, self.api_id, self.api_hash) - except OperationalError as e: - logger.error(f"Unable to access the {self.session_file} session, please make sure you don't use the same session file here and in telethon_archiver. if you do then disable at least one of the archivers for the 1st time you setup telethon session: {e}") @staticmethod def configs() -> dict: @@ -35,10 +33,30 @@ class InstagramTbotArchiver(Archiver): } def setup(self) -> None: + """ + 1. makes a copy of session_file that is removed in cleanup + 2. checks if the session file is valid + """ logger.info(f"SETUP {self.name} checking login...") + + # make a copy of the session that is used exclusively with this archiver instance + new_session_file = os.path.join("secrets/", f"instabot-{time.strftime('%Y-%m-%d')}{random_str(8)}.session") + shutil.copy(self.session_file + ".session", new_session_file) + self.session_file = new_session_file + + try: + self.client = TelegramClient(self.session_file, self.api_id, self.api_hash) + except OperationalError as e: + logger.error(f"Unable to access the {self.session_file} session, please make sure you don't use the same session file here and in telethon_archiver. if you do then disable at least one of the archivers for the 1st time you setup telethon session: {e}") + with self.client.start(): logger.success(f"SETUP {self.name} login works.") + def cleanup(self) -> None: + logger.info(f"CLEANUP {self.name}.") + if os.path.exists(self.session_file): + os.remove(self.session_file) + def download(self, item: Metadata) -> Metadata: url = item.get_url() if not "instagram.com" in url: return False diff --git a/src/auto_archiver/archivers/telegram_archiver.py b/src/auto_archiver/archivers/telegram_archiver.py index f93c66a..ed57927 100644 --- a/src/auto_archiver/archivers/telegram_archiver.py +++ b/src/auto_archiver/archivers/telegram_archiver.py @@ -53,10 +53,10 @@ class TelegramArchiver(Archiver): if not len(image_urls): return False for img_url in image_urls: - result.add_media(Media(self.download_from_url(img_url, item=item))) + result.add_media(Media(self.download_from_url(img_url))) else: video_url = video.get('src') - m_video = Media(self.download_from_url(video_url, item=item)) + m_video = Media(self.download_from_url(video_url)) # extract duration from HTML try: duration = s.find_all('time')[0].contents[0] diff --git a/src/auto_archiver/archivers/telethon_archiver.py b/src/auto_archiver/archivers/telethon_archiver.py index c54d026..9144c61 100644 --- a/src/auto_archiver/archivers/telethon_archiver.py +++ b/src/auto_archiver/archivers/telethon_archiver.py @@ -1,4 +1,5 @@ +import shutil from telethon.sync import TelegramClient from telethon.errors import ChannelInvalidError from telethon.tl.functions.messages import ImportChatInviteRequest @@ -9,6 +10,7 @@ import re, time, json, os from . import Archiver from ..core import Metadata, Media, ArchivingContext +from ..utils import random_str class TelethonArchiver(Archiver): @@ -21,8 +23,6 @@ class TelethonArchiver(Archiver): self.assert_valid_string("api_id") self.assert_valid_string("api_hash") - self.client = TelegramClient(self.session_file, self.api_id, self.api_hash) - @staticmethod def configs() -> dict: return { @@ -40,10 +40,20 @@ class TelethonArchiver(Archiver): def setup(self) -> None: """ - 1. trigger login process for telegram or proceed if already saved in a session file - 2. joins channel_invites where needed + 1. makes a copy of session_file that is removed in cleanup + 2. trigger login process for telegram or proceed if already saved in a session file + 3. joins channel_invites where needed """ logger.info(f"SETUP {self.name} checking login...") + + # make a copy of the session that is used exclusively with this archiver instance + new_session_file = os.path.join("secrets/", f"telethon-{time.strftime('%Y-%m-%d')}{random_str(8)}.session") + shutil.copy(self.session_file + ".session", new_session_file) + self.session_file = new_session_file + + # initiate the client + self.client = TelegramClient(self.session_file, self.api_id, self.api_hash) + with self.client.start(): logger.success(f"SETUP {self.name} login works.") @@ -89,6 +99,11 @@ class TelethonArchiver(Archiver): i += 1 pbar.update() + def cleanup(self) -> None: + logger.info(f"CLEANUP {self.name}.") + if os.path.exists(self.session_file): + os.remove(self.session_file) + def download(self, item: Metadata) -> Metadata: """ if this url is archivable will download post info and look for other posts from the same group with media. @@ -137,7 +152,7 @@ class TelethonArchiver(Archiver): if len(other_media_urls): logger.debug(f"Got {len(other_media_urls)} other media urls from {mp.id=}: {other_media_urls}") for i, om_url in enumerate(other_media_urls): - filename = self.download_from_url(om_url, f'{chat}_{group_id}_{i}', item) + filename = self.download_from_url(om_url, f'{chat}_{group_id}_{i}') result.add_media(Media(filename=filename), id=f"{group_id}_{i}") filename_dest = os.path.join(tmp_dir, f'{chat}_{group_id}', str(mp.id)) diff --git a/src/auto_archiver/archivers/twitter_api_archiver.py b/src/auto_archiver/archivers/twitter_api_archiver.py index e348cca..a8c4673 100644 --- a/src/auto_archiver/archivers/twitter_api_archiver.py +++ b/src/auto_archiver/archivers/twitter_api_archiver.py @@ -90,7 +90,7 @@ class TwitterApiArchiver(TwitterArchiver, Archiver): continue logger.info(f"Found media {media}") ext = mimetypes.guess_extension(mimetype) - media.filename = self.download_from_url(media.get("src"), f'{slugify(url)}_{i}{ext}', item) + media.filename = self.download_from_url(media.get("src"), f'{slugify(url)}_{i}{ext}') result.add_media(media) result.set_content(json.dumps({ diff --git a/src/auto_archiver/archivers/twitter_archiver.py b/src/auto_archiver/archivers/twitter_archiver.py index 787fa39..3e9efb3 100644 --- a/src/auto_archiver/archivers/twitter_archiver.py +++ b/src/auto_archiver/archivers/twitter_archiver.py @@ -80,7 +80,7 @@ class TwitterArchiver(Archiver): logger.warning(f"Could not get media URL of {tweet_media}") continue ext = mimetypes.guess_extension(mimetype) - media.filename = self.download_from_url(media.get("src"), f'{slugify(url)}_{i}{ext}', item) + media.filename = self.download_from_url(media.get("src"), f'{slugify(url)}_{i}{ext}') result.add_media(media) return result.success("twitter-snscrape") @@ -120,7 +120,7 @@ class TwitterArchiver(Archiver): if (mtype := mimetypes.guess_type(UrlUtil.remove_get_parameters(u))[0]): ext = mimetypes.guess_extension(mtype) - media.filename = self.download_from_url(u, f'{slugify(url)}_{i}{ext}', item) + media.filename = self.download_from_url(u, f'{slugify(url)}_{i}{ext}') result.add_media(media) result.set_title(tweet.get("text")).set_content(json.dumps(tweet, ensure_ascii=False)).set_timestamp(datetime.strptime(tweet["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ")) diff --git a/src/auto_archiver/archivers/youtubedl_archiver.py b/src/auto_archiver/archivers/youtubedl_archiver.py index 0184737..fb10d0d 100644 --- a/src/auto_archiver/archivers/youtubedl_archiver.py +++ b/src/auto_archiver/archivers/youtubedl_archiver.py @@ -1,4 +1,4 @@ -import datetime, os, yt_dlp +import datetime, os, yt_dlp, pysubs2 from loguru import logger from . import Archiver @@ -10,38 +10,51 @@ class YoutubeDLArchiver(Archiver): def __init__(self, config: dict) -> None: super().__init__(config) + self.subtitles = bool(self.subtitles) + self.comments = bool(self.comments) + self.livestreams = bool(self.livestreams) + self.live_from_start = bool(self.live_from_start) + self.end_means_success = bool(self.end_means_success) @staticmethod def configs() -> dict: return { "facebook_cookie": {"default": None, "help": "optional facebook cookie to have more access to content, from browser, looks like 'cookie: datr= xxxx'"}, + "subtitles": {"default": True, "help": "download subtitles if available"}, + "comments": {"default": False, "help": "download all comments if available, may lead to large metadata"}, + "livestreams": {"default": False, "help": "if set, will download live streams, otherwise will skip them; see --max-filesize for more control"}, + "live_from_start": {"default": False, "help": "if set, will download live streams from their earliest available moment, otherwise starts now."}, + "proxy": {"default": "", "help": "http/socks (https seems to not work atm) proxy to use for the webdriver, eg https://proxy-user:password@proxy-ip:port"}, + "end_means_success": {"default": True, "help": "if True, any archived content will mean a 'success', if False this archiver will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent archivers can retrieve."}, } def download(self, item: Metadata) -> Metadata: - #TODO: yt-dlp for transcripts? url = item.get_url() if item.netloc in ['facebook.com', 'www.facebook.com'] and self.facebook_cookie: logger.debug('Using Facebook cookie') yt_dlp.utils.std_headers['cookie'] = self.facebook_cookie - ydl = yt_dlp.YoutubeDL({'outtmpl': os.path.join(ArchivingContext.get_tmp_dir(), f'%(id)s.%(ext)s'), 'quiet': False, 'noplaylist': True}) + ydl_options = {'outtmpl': os.path.join(ArchivingContext.get_tmp_dir(), f'%(id)s.%(ext)s'), 'quiet': False, 'noplaylist': True, 'writesubtitles': self.subtitles, 'writeautomaticsub': self.subtitles, "live_from_start": self.live_from_start, "proxy": self.proxy} + ydl = yt_dlp.YoutubeDL(ydl_options) # allsubtitles and subtitleslangs not working as expected, so default lang is always "en" try: # don'd download since it can be a live stream info = ydl.extract_info(url, download=False) - if info.get('is_live', False): - logger.warning("Live streaming media, not archiving now") + if info.get('is_live', False) and not self.livestreams: + logger.warning("Livestream detected, skipping due to 'livestreams' configuration setting") return False except yt_dlp.utils.DownloadError as e: logger.debug(f'No video - Youtube normal control flow: {e}') return False except Exception as e: - logger.debug(f'ytdlp exception which is normal for example a facebook page with images only will cause a IndexError: list index out of range. Exception here is: \n {e}') + logger.debug(f'ytdlp exception which is normal for example a facebook page with images only will cause a IndexError: list index out of range. Exception is: \n {e}') return False # this time download + ydl = yt_dlp.YoutubeDL({**ydl_options, "getcomments": self.comments}) info = ydl.extract_info(url, download=True) + if "entries" in info: entries = info.get("entries", []) if not len(entries): @@ -52,10 +65,32 @@ class YoutubeDLArchiver(Archiver): result = Metadata() result.set_title(info.get("title")) for entry in entries: - filename = ydl.prepare_filename(entry) - if not os.path.exists(filename): - filename = filename.split('.')[0] + '.mkv' - result.add_media(Media(filename).set("duration", info.get("duration"))) + try: + filename = ydl.prepare_filename(entry) + if not os.path.exists(filename): + filename = filename.split('.')[0] + '.mkv' + new_media = Media(filename).set("duration", info.get("duration")) + + # read text from subtitles if enabled + if self.subtitles: + for lang, val in (info.get('requested_subtitles') or {}).items(): + try: + subs = pysubs2.load(val.get('filepath'), encoding="utf-8") + text = " ".join([line.text for line in subs]) + new_media.set(f"subtitles_{lang}", text) + except Exception as e: + logger.error(f"Error loading subtitle file {val.get('filepath')}: {e}") + result.add_media(new_media) + except Exception as e: + logger.error(f"Error processing entry {entry}: {e}") + + # extract comments if enabled + if self.comments: + result.set("comments", [{ + "text": c["text"], + "author": c["author"], + "timestamp": datetime.datetime.utcfromtimestamp(c.get("timestamp")).replace(tzinfo=datetime.timezone.utc) + } for c in info.get("comments", [])]) if (timestamp := info.get("timestamp")): timestamp = datetime.datetime.utcfromtimestamp(timestamp).replace(tzinfo=datetime.timezone.utc).isoformat() @@ -64,4 +99,6 @@ class YoutubeDLArchiver(Archiver): upload_date = datetime.datetime.strptime(upload_date, '%Y%m%d').replace(tzinfo=datetime.timezone.utc) result.set("upload_date", upload_date) - return result.success("yt-dlp") + if self.end_means_success: result.success("yt-dlp") + else: result.status = "yt-dlp" + return result diff --git a/src/auto_archiver/core/context.py b/src/auto_archiver/core/context.py index f836c67..8fdf040 100644 --- a/src/auto_archiver/core/context.py +++ b/src/auto_archiver/core/context.py @@ -1,6 +1,3 @@ -from loguru import logger - - class ArchivingContext: """ Singleton context class. diff --git a/src/auto_archiver/core/media.py b/src/auto_archiver/core/media.py index 51ad9c2..7b18b27 100644 --- a/src/auto_archiver/core/media.py +++ b/src/auto_archiver/core/media.py @@ -44,10 +44,14 @@ class Media: """ if include_self: yield self for prop in self.properties.values(): - if isinstance(prop, Media): yield prop + if isinstance(prop, Media): + for inner_media in prop.all_inner_media(include_self=True): + yield inner_media if isinstance(prop, list): for prop_media in prop: - if isinstance(prop_media, Media): yield prop_media + if isinstance(prop_media, Media): + for inner_media in prop_media.all_inner_media(include_self=True): + yield inner_media def is_stored(self) -> bool: return len(self.urls) > 0 and len(self.urls) == len(ArchivingContext.get("storages")) diff --git a/src/auto_archiver/core/metadata.py b/src/auto_archiver/core/metadata.py index cca49a4..5e2182a 100644 --- a/src/auto_archiver/core/metadata.py +++ b/src/auto_archiver/core/metadata.py @@ -54,6 +54,12 @@ class Metadata: self.metadata[key] = val return self + def append(self, key: str, val: Any) -> Metadata: + if key not in self.metadata: + self.metadata[key] = [] + self.metadata[key] = val + return self + def get(self, key: str, default: Any = None, create_if_missing=False) -> Union[Metadata, str]: # goes through metadata and returns the Metadata available if create_if_missing and key not in self.metadata: @@ -69,7 +75,8 @@ class Metadata: return "success" in self.status def is_empty(self) -> bool: - return not self.is_success() and len(self.media) == 0 and len(self.metadata) <= 2 # url, processed_at + meaningfull_ids = set(self.metadata.keys()) - set(["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"]) + return not self.is_success() and len(self.media) == 0 and len(meaningfull_ids) == 0 @property # getter .netloc def netloc(self) -> str: diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py index fb5e2ab..bbdf37c 100644 --- a/src/auto_archiver/core/orchestrator.py +++ b/src/auto_archiver/core/orchestrator.py @@ -25,13 +25,28 @@ class ArchivingOrchestrator: self.storages: List[Storage] = config.storages ArchivingContext.set("storages", self.storages, keep_on_reset=True) - for a in self.archivers: a.setup() + try: + for a in self.archivers: a.setup() + except (KeyboardInterrupt, Exception) as e: + logger.error(f"Error during setup of archivers: {e}\n{traceback.format_exc()}") + self.cleanup() + + + def cleanup(self)->None: + logger.info("Cleaning up") + for a in self.archivers: a.cleanup() def feed(self) -> Generator[Metadata]: for item in self.feeder: yield self.feed_item(item) + self.cleanup() def feed_item(self, item: Metadata) -> Metadata: + """ + Takes one item (URL) to archive and calls self.archive, additionally: + - catches keyboard interruptions to do a clean exit + - catches any unexpected error, logs it, and does a clean exit + """ try: ArchivingContext.reset() with tempfile.TemporaryDirectory(dir="./") as tmp_dir: @@ -41,36 +56,34 @@ class ArchivingOrchestrator: # catches keyboard interruptions to do a clean exit logger.warning(f"caught interrupt on {item=}") for d in self.databases: d.aborted(item) + self.cleanup() exit() except Exception as e: logger.error(f'Got unexpected error on item {item}: {e}\n{traceback.format_exc()}') for d in self.databases: d.failed(item) - # how does this handle the parameters like folder which can be different for each archiver? - # the storage needs to know where to archive!! - # solution: feeders have context: extra metadata that they can read or ignore, - # all of it should have sensible defaults (eg: folder) - # default feeder is a list with 1 element def archive(self, result: Metadata) -> Union[Metadata, None]: + """ + Runs the archiving process for a single URL + 1. Each archiver can sanitize its own URLs + 2. Check for cached results in Databases, and signal start to the databases + 3. Call Archivers until one succeeds + 4. Call Enrichers + 5. Store all downloaded/generated media + 6. Call selected Formatter and store formatted if needed + """ original_url = result.get_url() - # 1 - cleanup - # each archiver is responsible for cleaning/expanding its own URLs + # 1 - sanitize - each archiver is responsible for cleaning/expanding its own URLs url = original_url for a in self.archivers: url = a.sanitize_url(url) result.set_url(url) if original_url != url: result.set("original_url", original_url) - # 2 - notify start to DB - # signal to DB that archiving has started - # and propagate already archived if it exists + # 2 - notify start to DBs, propagate already archived if feature enabled in DBs cached_result = None for d in self.databases: - # are the databases to decide whether to archive? - # they can simply return True by default, otherwise they can avoid duplicates. should this logic be more granular, for example on the archiver level: a tweet will not need be scraped twice, whereas an instagram profile might. the archiver could not decide from the link which parts to archive, - # instagram profile example: it would always re-archive everything - # maybe the database/storage could use a hash/key to decide if there's a need to re-archive d.started(result) if (local_result := d.fetch(result)): cached_result = (cached_result or Metadata()).merge(local_result) @@ -84,30 +97,21 @@ class ArchivingOrchestrator: for a in self.archivers: logger.info(f"Trying archiver {a.name} for {url}") try: - # Q: should this be refactored so it's just a.download(result)? result.merge(a.download(result)) if result.is_success(): break - except Exception as e: logger.error(f"Unexpected error with archiver {a.name}: {e}: {traceback.format_exc()}") + except Exception as e: + logger.error(f"ERROR archiver {a.name}: {e}: {traceback.format_exc()}") - # what if an archiver returns multiple entries and one is to be part of HTMLgenerator? - # should it call the HTMLgenerator as if it's not an enrichment? - # eg: if it is enable: generates an HTML with all the returned media, should it include enrichers? yes - # then how to execute it last? should there also be post-processors? are there other examples? - # maybe as a PDF? or a Markdown file - - # 4 - call enrichers: have access to archived content, can generate metadata and Media - # eg: screenshot, wacz, webarchive, thumbnails + # 4 - call enrichers to work with archived content for e in self.enrichers: try: e.enrich(result) - except Exception as exc: logger.error(f"Unexpected error with enricher {e.name}: {exc}: {traceback.format_exc()}") + except Exception as exc: + logger.error(f"ERROR enricher {e.name}: {exc}: {traceback.format_exc()}") - # 5 - store media - # looks for Media in result.media and also result.media[x].properties (as list or dict values) + # 5 - store all downloaded/generated media result.store() - # 6 - format and store formatted if needed - # enrichers typically need access to already stored URLs etc if (final_media := self.formatter.format(result)): final_media.store(url=url) result.set_final_media(final_media) @@ -115,7 +119,7 @@ class ArchivingOrchestrator: if result.is_empty(): result.status = "nothing archived" - # signal completion to databases (DBs, Google Sheets, CSV, ...) + # signal completion to databases and archivers for d in self.databases: d.done(result) return result diff --git a/src/auto_archiver/databases/database.py b/src/auto_archiver/databases/database.py index 1ea1982..30e23fc 100644 --- a/src/auto_archiver/databases/database.py +++ b/src/auto_archiver/databases/database.py @@ -32,7 +32,7 @@ class Database(Step, ABC): # @abstractmethod def fetch(self, item: Metadata) -> Union[Metadata, bool]: - """check if the given item has been archived already""" + """check and fetch if the given item has been archived already, each database should handle its own caching, and configuration mechanisms""" return False @abstractmethod diff --git a/src/auto_archiver/enrichers/__init__.py b/src/auto_archiver/enrichers/__init__.py index 9f78b39..5681403 100644 --- a/src/auto_archiver/enrichers/__init__.py +++ b/src/auto_archiver/enrichers/__init__.py @@ -7,4 +7,6 @@ from .wacz_enricher import WaczArchiverEnricher from .whisper_enricher import WhisperEnricher from .pdq_hash_enricher import PdqHashEnricher from .metadata_enricher import MetadataEnricher -from .meta_enricher import MetaEnricher \ No newline at end of file +from .meta_enricher import MetaEnricher +from .ssl_enricher import SSLEnricher +from .timestamping_enricher import TimestampingEnricher \ No newline at end of file diff --git a/src/auto_archiver/enrichers/meta_enricher.py b/src/auto_archiver/enrichers/meta_enricher.py index 4c22f75..0d84b9c 100644 --- a/src/auto_archiver/enrichers/meta_enricher.py +++ b/src/auto_archiver/enrichers/meta_enricher.py @@ -19,22 +19,26 @@ class MetaEnricher(Enricher): @staticmethod def configs() -> dict: - return { - } + return {} def enrich(self, to_enrich: Metadata) -> None: - logger.debug(f"calculating archive metadata information for url={to_enrich.get_url()}") + url = to_enrich.get_url() + if to_enrich.is_empty(): + logger.debug(f"[SKIP] META_ENRICHER there is no media or metadata to enrich: {url=}") + return + logger.debug(f"calculating archive metadata information for {url=}") + self.enrich_file_sizes(to_enrich) self.enrich_archive_duration(to_enrich) - def enrich_file_sizes(self, to_enrich): + def enrich_file_sizes(self, to_enrich: Metadata): logger.debug(f"calculating archive file sizes for url={to_enrich.get_url()} ({len(to_enrich.media)} media files)") total_size = 0 - for i, m in enumerate(to_enrich.media): - file_stats = os.stat(m.filename) - to_enrich.media[i].set("bytes", file_stats.st_size) - to_enrich.media[i].set("size", self.human_readable_bytes(file_stats.st_size)) + for media in to_enrich.get_all_media(): + file_stats = os.stat(media.filename) + media.set("bytes", file_stats.st_size) + media.set("size", self.human_readable_bytes(file_stats.st_size)) total_size += file_stats.st_size to_enrich.set("total_bytes", total_size) diff --git a/src/auto_archiver/enrichers/screenshot_enricher.py b/src/auto_archiver/enrichers/screenshot_enricher.py index eaad5ff..69f466b 100644 --- a/src/auto_archiver/enrichers/screenshot_enricher.py +++ b/src/auto_archiver/enrichers/screenshot_enricher.py @@ -16,7 +16,8 @@ class ScreenshotEnricher(Enricher): "width": {"default": 1280, "help": "width of the screenshots"}, "height": {"default": 720, "help": "height of the screenshots"}, "timeout": {"default": 60, "help": "timeout for taking the screenshot"}, - "sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"} + "sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"}, + "http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"}, } def enrich(self, to_enrich: Metadata) -> None: @@ -26,7 +27,7 @@ class ScreenshotEnricher(Enricher): return logger.debug(f"Enriching screenshot for {url=}") - with Webdriver(self.width, self.height, self.timeout, 'facebook.com' in url) as driver: + with Webdriver(self.width, self.height, self.timeout, 'facebook.com' in url, http_proxy=self.http_proxy) as driver: try: driver.get(url) time.sleep(int(self.sleep_before_screenshot)) diff --git a/src/auto_archiver/enrichers/ssl_enricher.py b/src/auto_archiver/enrichers/ssl_enricher.py new file mode 100644 index 0000000..9a06f71 --- /dev/null +++ b/src/auto_archiver/enrichers/ssl_enricher.py @@ -0,0 +1,36 @@ +import ssl, os +from slugify import slugify +from urllib.parse import urlparse +from loguru import logger + +from . import Enricher +from ..core import Metadata, ArchivingContext, Media + + +class SSLEnricher(Enricher): + """ + Retrieves SSL certificate information for a domain, as a file + """ + name = "ssl_enricher" + + def __init__(self, config: dict) -> None: + super().__init__(config) + self. skip_when_nothing_archived = bool(self.skip_when_nothing_archived) + + @staticmethod + def configs() -> dict: + return { + "skip_when_nothing_archived": {"default": True, "help": "if true, will skip enriching when no media is archived"}, + } + + def enrich(self, to_enrich: Metadata) -> None: + if not to_enrich.media and self.skip_when_nothing_archived: return + + url = to_enrich.get_url() + domain = urlparse(url).netloc + logger.debug(f"fetching SSL certificate for {domain=} in {url=}") + + cert = ssl.get_server_certificate((domain, 443)) + cert_fn = os.path.join(ArchivingContext.get_tmp_dir(), f"{slugify(domain)}.pem") + with open(cert_fn, "w") as f: f.write(cert) + to_enrich.add_media(Media(filename=cert_fn), id="ssl_certificate") diff --git a/src/auto_archiver/enrichers/thumbnail_enricher.py b/src/auto_archiver/enrichers/thumbnail_enricher.py index 8e24389..0ffbe38 100644 --- a/src/auto_archiver/enrichers/thumbnail_enricher.py +++ b/src/auto_archiver/enrichers/thumbnail_enricher.py @@ -15,32 +15,54 @@ class ThumbnailEnricher(Enricher): def __init__(self, config: dict) -> None: # without this STEP.__init__ is not called super().__init__(config) + self.thumbnails_per_second = int(self.thumbnails_per_minute) / 60 + self.max_thumbnails = int(self.max_thumbnails) @staticmethod def configs() -> dict: - return {} - + return { + "thumbnails_per_minute": {"default": 60, "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails"}, + "max_thumbnails": {"default": 16, "help": "limit the number of thumbnails to generate per video, 0 means no limit"}, + } + def enrich(self, to_enrich: Metadata) -> None: - logger.debug(f"generating thumbnails") - for i, m in enumerate(to_enrich.media[::]): + """ + Uses or reads the video duration to generate thumbnails + Calculates how many thumbnails to generate and at which timestamps based on the video duration, the number of thumbnails per minute and the max number of thumbnails. + Thumbnails are equally distributed across the video duration. + """ + logger.debug(f"generating thumbnails for {to_enrich.get_url()}") + for m_id, m in enumerate(to_enrich.media[::]): if m.is_video(): folder = os.path.join(ArchivingContext.get_tmp_dir(), random_str(24)) os.makedirs(folder, exist_ok=True) logger.debug(f"generating thumbnails for {m.filename}") - fps, duration = 0.5, m.get("duration") - if duration is not None: - duration = float(duration) - if duration < 60: fps = 10.0 / duration - elif duration < 120: fps = 20.0 / duration - else: fps = 40.0 / duration + duration = m.get("duration") - stream = ffmpeg.input(m.filename) - stream = ffmpeg.filter(stream, 'fps', fps=fps).filter('scale', 512, -1) - stream.output(os.path.join(folder, 'out%d.jpg')).run() + if duration is None: + try: + probe = ffmpeg.probe(m.filename) + duration = float(next(stream for stream in probe['streams'] if stream['codec_type'] == 'video')['duration']) + to_enrich.media[m_id].set("duration", duration) + except Exception as e: + logger.error(f"error getting duration of video {m.filename}: {e}") + return + + num_thumbs = int(min(max(1, duration * self.thumbnails_per_second), self.max_thumbnails)) + timestamps = [duration / (num_thumbs + 1) * i for i in range(1, num_thumbs + 1)] - thumbnails = os.listdir(folder) thumbnails_media = [] - for t, fname in enumerate(thumbnails): - if fname[-3:] == 'jpg': - thumbnails_media.append(Media(filename=os.path.join(folder, fname)).set("id", f"thumbnail_{t}")) - to_enrich.media[i].set("thumbnails", thumbnails_media) + for index, timestamp in enumerate(timestamps): + output_path = os.path.join(folder, f"out{index}.jpg") + ffmpeg.input(m.filename, ss=timestamp).filter('scale', 512, -1).output(output_path, vframes=1, loglevel="quiet").run() + + try: + thumbnails_media.append(Media( + filename=output_path) + .set("id", f"thumbnail_{index}") + .set("timestamp", "%.3fs" % timestamp) + ) + except Exception as e: + logger.error(f"error creating thumbnail {index} for media: {e}") + + to_enrich.media[m_id].set("thumbnails", thumbnails_media) diff --git a/src/auto_archiver/enrichers/timestamping_enricher.py b/src/auto_archiver/enrichers/timestamping_enricher.py new file mode 100644 index 0000000..dffa1a3 --- /dev/null +++ b/src/auto_archiver/enrichers/timestamping_enricher.py @@ -0,0 +1,136 @@ +import os +from loguru import logger +from tsp_client import TSPSigner, SigningSettings, TSPVerifier +from tsp_client.algorithms import DigestAlgorithm +from importlib.metadata import version +from asn1crypto.cms import ContentInfo +from certvalidator import CertificateValidator, ValidationContext +from asn1crypto import pem +import certifi + +from . import Enricher +from ..core import Metadata, ArchivingContext, Media +from ..archivers import Archiver + + +class TimestampingEnricher(Enricher): + """ + Uses several RFC3161 Time Stamp Authorities to generate a timestamp token that will be preserved. This can be used to prove that a certain file existed at a certain time, useful for legal purposes, for example, to prove that a certain file was not tampered with after a certain date. + + The information that gets timestamped is concatenation (via paragraphs) of the file hashes existing in the current archive. It will depend on which archivers and enrichers ran before this one. Inner media files (like thumbnails) are not included in the .txt file. It should run AFTER the hash_enricher. + + See https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710 for list of timestamp authorities. + """ + name = "timestamping_enricher" + + def __init__(self, config: dict) -> None: + super().__init__(config) + + @staticmethod + def configs() -> dict: + return { + "tsa_urls": { + "default": [ + # [Adobe Approved Trust List] and [Windows Cert Store] + "http://timestamp.digicert.com", + "http://timestamp.identrust.com", + # "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping + # "https://timestamp.sectigo.com", # wait 15 seconds between each request. + + # [Adobe: European Union Trusted Lists]. + # "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request. + + # [Windows Cert Store] + "http://timestamp.globalsign.com/tsa/r6advanced1", + + # [Adobe: European Union Trusted Lists] and [Windows Cert Store] + # "http://ts.quovadisglobal.com/eu", # not valid for timestamping + # "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain + # "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain + # "http://tsa.sep.bg", # self-signed certificate in certificate chain + # "http://tsa.izenpe.com", #unable to get local issuer certificate + # "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate + "http://tss.accv.es:8318/tsa", + ], + "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.", + "cli_set": lambda cli_val, cur_val: set(cli_val.split(",")) + } + } + + def enrich(self, to_enrich: Metadata) -> None: + url = to_enrich.get_url() + logger.debug(f"RFC3161 timestamping existing files for {url=}") + + # create a new text file with the existing media hashes + hashes = [m.get("hash").replace("SHA-256:", "").replace("SHA3-512:", "") for m in to_enrich.media if m.get("hash")] + + if not len(hashes): + logger.warning(f"No hashes found in {url=}") + return + + tmp_dir = ArchivingContext.get_tmp_dir() + hashes_fn = os.path.join(tmp_dir, "hashes.txt") + + data_to_sign = "\n".join(hashes) + with open(hashes_fn, "w") as f: + f.write(data_to_sign) + hashes_media = Media(filename=hashes_fn) + + timestamp_tokens = [] + from slugify import slugify + for tsa_url in self.tsa_urls: + try: + signing_settings = SigningSettings(tsp_server=tsa_url, digest_algorithm=DigestAlgorithm.SHA256) + signer = TSPSigner() + message = bytes(data_to_sign, encoding='utf8') + # send TSQ and get TSR from the TSA server + signed = signer.sign(message=message, signing_settings=signing_settings) + # fail if there's any issue with the certificates, uses certifi list of trusted CAs + TSPVerifier(certifi.where()).verify(signed, message=message) + # download and verify timestamping certificate + cert_chain = self.download_and_verify_certificate(signed) + # continue with saving the timestamp token + tst_fn = os.path.join(tmp_dir, f"timestamp_token_{slugify(tsa_url)}") + with open(tst_fn, "wb") as f: f.write(signed) + timestamp_tokens.append(Media(filename=tst_fn).set("tsa", tsa_url).set("cert_chain", cert_chain)) + except Exception as e: + logger.warning(f"Error while timestamping {url=} with {tsa_url=}: {e}") + + if len(timestamp_tokens): + hashes_media.set("timestamp_authority_files", timestamp_tokens) + hashes_media.set("certifi v", version("certifi")) + hashes_media.set("tsp_client v", version("tsp_client")) + hashes_media.set("certvalidator v", version("certvalidator")) + to_enrich.add_media(hashes_media, id="timestamped_hashes") + to_enrich.set("timestamped", True) + logger.success(f"{len(timestamp_tokens)} timestamp tokens created for {url=}") + else: + logger.warning(f"No successful timestamps for {url=}") + + def download_and_verify_certificate(self, signed: bytes) -> list[Media]: + # returns the leaf certificate URL, fails if not set + tst = ContentInfo.load(signed) + + trust_roots = [] + with open(certifi.where(), 'rb') as f: + for _, _, der_bytes in pem.unarmor(f.read(), multiple=True): + trust_roots.append(der_bytes) + context = ValidationContext(trust_roots=trust_roots) + + certificates = tst["content"]["certificates"] + first_cert = certificates[0].dump() + intermediate_certs = [] + for i in range(1, len(certificates)): # cannot use list comprehension [1:] + intermediate_certs.append(certificates[i].dump()) + + validator = CertificateValidator(first_cert, intermediate_certs=intermediate_certs, validation_context=context) + path = validator.validate_usage({'digital_signature'}, extended_key_usage={'time_stamping'}) + + cert_chain = [] + for cert in path: + cert_fn = os.path.join(ArchivingContext.get_tmp_dir(), f"{str(cert.serial_number)[:20]}.crt") + with open(cert_fn, "wb") as f: + f.write(cert.dump()) + cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.native["common_name"])) + + return cert_chain \ No newline at end of file diff --git a/src/auto_archiver/enrichers/wacz_enricher.py b/src/auto_archiver/enrichers/wacz_enricher.py index 86cdd3a..0acd885 100644 --- a/src/auto_archiver/enrichers/wacz_enricher.py +++ b/src/auto_archiver/enrichers/wacz_enricher.py @@ -31,7 +31,9 @@ class WaczArchiverEnricher(Enricher, Archiver): "docker_commands": {"default": None, "help":"if a custom docker invocation is needed"}, "timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"}, "extract_media": {"default": False, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, - "extract_screenshot": {"default": True, "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."} + "extract_screenshot": {"default": True, "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, + "socks_proxy_host": {"default": None, "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host"}, + "socks_proxy_port": {"default": None, "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"}, } def download(self, item: Metadata) -> Metadata: @@ -91,7 +93,12 @@ class WaczArchiverEnricher(Enricher, Archiver): try: logger.info(f"Running browsertrix-crawler: {' '.join(cmd)}") - subprocess.run(cmd, check=True) + if self.socks_proxy_host and self.socks_proxy_port: + logger.debug("Using SOCKS proxy for browsertrix-crawler") + my_env = os.environ.copy() + my_env["SOCKS_HOST"] = self.socks_proxy_host + my_env["SOCKS_PORT"] = str(self.socks_proxy_port) + subprocess.run(cmd, check=True, env=my_env) except Exception as e: logger.error(f"WACZ generation failed: {e}") return False @@ -191,7 +198,7 @@ class WaczArchiverEnricher(Enricher, Archiver): # if a link with better quality exists, try to download that if record_url_best_qual != record_url: try: - m.filename = self.download_from_url(record_url_best_qual, warc_fn, to_enrich) + m.filename = self.download_from_url(record_url_best_qual, warc_fn) m.set("src", record_url_best_qual) m.set("src_alternative", record_url) except Exception as e: logger.warning(f"Unable to download best quality URL for {record_url=} got error {e}, using original in WARC.") diff --git a/src/auto_archiver/enrichers/wayback_enricher.py b/src/auto_archiver/enrichers/wayback_enricher.py index cb8107a..12eb6e3 100644 --- a/src/auto_archiver/enrichers/wayback_enricher.py +++ b/src/auto_archiver/enrichers/wayback_enricher.py @@ -1,7 +1,6 @@ from loguru import logger import time, requests - from . import Enricher from ..archivers import Archiver from ..utils import UrlUtil @@ -9,7 +8,9 @@ from ..core import Metadata class WaybackArchiverEnricher(Enricher, Archiver): """ - Submits the current URL to the webarchive and returns a job_id or completed archive + Submits the current URL to the webarchive and returns a job_id or completed archive. + + The Wayback machine will rate-limit IP heavy usage. """ name = "wayback_archiver_enricher" @@ -25,7 +26,9 @@ class WaybackArchiverEnricher(Enricher, Archiver): "timeout": {"default": 15, "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually."}, "if_not_archived_within": {"default": None, "help": "only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA"}, "key": {"default": None, "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php"}, - "secret": {"default": None, "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php"} + "secret": {"default": None, "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php"}, + "proxy_http": {"default": None, "help": "http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port"}, + "proxy_https": {"default": None, "help": "https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port"}, } def download(self, item: Metadata) -> Metadata: @@ -36,6 +39,10 @@ class WaybackArchiverEnricher(Enricher, Archiver): return result.success("wayback") def enrich(self, to_enrich: Metadata) -> bool: + proxies = {} + if self.proxy_http: proxies["http"] = self.proxy_http + if self.proxy_https: proxies["https"] = self.proxy_https + url = to_enrich.get_url() if UrlUtil.is_auth_wall(url): logger.debug(f"[SKIP] WAYBACK since url is behind AUTH WALL: {url=}") @@ -55,7 +62,7 @@ class WaybackArchiverEnricher(Enricher, Archiver): if self.if_not_archived_within: post_data["if_not_archived_within"] = self.if_not_archived_within # see https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA for more options - r = requests.post('https://web.archive.org/save/', headers=ia_headers, data=post_data) + r = requests.post('https://web.archive.org/save/', headers=ia_headers, data=post_data, proxies=proxies) if r.status_code != 200: logger.error(em := f"Internet archive failed with status of {r.status_code}: {r.json()}") @@ -75,14 +82,16 @@ class WaybackArchiverEnricher(Enricher, Archiver): while not wayback_url and time.time() - start_time <= self.timeout: try: logger.debug(f"GETting status for {job_id=} on {url=} ({attempt=})") - r_status = requests.get(f'https://web.archive.org/save/status/{job_id}', headers=ia_headers) + r_status = requests.get(f'https://web.archive.org/save/status/{job_id}', headers=ia_headers, proxies=proxies) r_json = r_status.json() if r_status.status_code == 200 and r_json['status'] == 'success': wayback_url = f"https://web.archive.org/web/{r_json['timestamp']}/{r_json['original_url']}" elif r_status.status_code != 200 or r_json['status'] != 'pending': logger.error(f"Wayback failed with {r_json}") return False - + except requests.exceptions.RequestException as e: + logger.warning(f"RequestException: fetching status for {url=} due to: {e}") + break except Exception as e: logger.warning(f"error fetching status for {url=} due to: {e}") if not wayback_url: diff --git a/src/auto_archiver/enrichers/whisper_enricher.py b/src/auto_archiver/enrichers/whisper_enricher.py index 22dc6e7..7ceab54 100644 --- a/src/auto_archiver/enrichers/whisper_enricher.py +++ b/src/auto_archiver/enrichers/whisper_enricher.py @@ -10,7 +10,7 @@ from ..storages import S3Storage class WhisperEnricher(Enricher): """ Connects with a Whisper API service to get texts out of audio - whisper API repository: TODO + whisper API repository: https://github.com/bellingcat/whisperbox-transcribe/ Only works if an S3 compatible storage is used """ name = "whisper_enricher" diff --git a/src/auto_archiver/formatters/html_formatter.py b/src/auto_archiver/formatters/html_formatter.py index 7202a0b..da79ab0 100644 --- a/src/auto_archiver/formatters/html_formatter.py +++ b/src/auto_archiver/formatters/html_formatter.py @@ -4,6 +4,8 @@ import mimetypes, os, pathlib from jinja2 import Environment, FileSystemLoader from urllib.parse import quote from loguru import logger +import minify_html, json +import base64 from ..version import __version__ from ..core import Metadata, Media, ArchivingContext @@ -45,6 +47,8 @@ class HtmlFormatter(Formatter): metadata=item.metadata, version=__version__ ) + content = minify_html.minify(content, minify_js=False, minify_css=True) + html_path = os.path.join(ArchivingContext.get_tmp_dir(), f"formatted{random_str(24)}.html") with open(html_path, mode="w", encoding="utf-8") as outf: outf.write(content) @@ -89,3 +93,8 @@ class JinjaHelpers: @staticmethod def quote(s: str) -> str: return quote(s) + + @staticmethod + def json_dump_b64(d: dict) -> str: + j = json.dumps(d, indent=4, default=str) + return base64.b64encode(j.encode()).decode() diff --git a/src/auto_archiver/formatters/templates/html_template.html b/src/auto_archiver/formatters/templates/html_template.html index eb23bf1..3dbe7be 100644 --- a/src/auto_archiver/formatters/templates/html_template.html +++ b/src/auto_archiver/formatters/templates/html_template.html @@ -96,6 +96,16 @@ overflow: hidden; background-color: #f1f1f1; } + + .pem-certificate, .text-preview { + text-align: left; + font-size: small; + } + .text-preview{ + padding-left: 10px; + padding-right: 10px; + white-space: pre-wrap; + } @@ -121,42 +131,7 @@ {% for m in media %} - + {{ macros.display_recursive(m, true) }} {{ macros.display_media(m, true, url) }} @@ -175,16 +150,68 @@ {{ key }} + {% if metadata[key] is mapping %} +
Copy as JSON
+ {% endif %} {{ macros.copy_urlize(metadata[key]) }} {% endfor %} -

Made with bellingcat/auto-archiver v{{ version }}

+

Made with bellingcat/auto-archiver + v{{ version }}

+ \ No newline at end of file diff --git a/src/auto_archiver/formatters/templates/macros.html b/src/auto_archiver/formatters/templates/macros.html index 50c3673..c2281c2 100644 --- a/src/auto_archiver/formatters/templates/macros.html +++ b/src/auto_archiver/formatters/templates/macros.html @@ -18,6 +18,12 @@ No URL available for {{ m.key }}. BingTineye + +
+ Image Forensics:  + FotoForensics,  + Media Verification Assistant +

{% elif 'video' in m.mimetype %} @@ -35,8 +41,15 @@ No URL available for {{ m.key }}. {% elif m.filename | get_extension == ".wacz" %} replayweb + +{% elif m.filename | get_extension == ".pem" %} + + +{% elif 'text' in m.mimetype %} +
PREVIEW:
+ {% else %} -No preview available for {{ m.key }}. +No preview available for {{ m.key }}. {% endif %} {% else %} {{ m.url | urlize }} @@ -54,7 +67,12 @@ No preview available for {{ m.key }}. {% macro copy_urlize(val, href_text) -%} -{% if val is mapping %} +{% if val | is_list %} + {% for item in val %} + {{ copy_urlize(item) }} + {% endfor %} + +{% elif val is mapping %} {% else %} -{% if href_text | length == 0 %} +{% if href_text | length == 0 %} {{ val | string | urlize }} {% else %} {{ href_text | string | urlize }} {% endif %} {% endif %} +{%- endmacro -%} + + +{% macro display_recursive(prop, skip_display) -%} + {% if prop is mapping %} +
Copy as JSON
+ + + {% elif prop | is_list %} + {% for item in prop %} +
  • + {{ display_recursive(item) }} +
  • + {% endfor %} + + + {% elif prop | is_media %} + {% if not skip_display %} + {{ display_media(prop, true) }} + {% endif %} + + {% else %} + {{ copy_urlize(prop) }} + {% endif %} {%- endmacro -%} \ No newline at end of file diff --git a/src/auto_archiver/utils/webdriver.py b/src/auto_archiver/utils/webdriver.py index 8d54f96..dc21e17 100644 --- a/src/auto_archiver/utils/webdriver.py +++ b/src/auto_archiver/utils/webdriver.py @@ -1,21 +1,24 @@ from __future__ import annotations from selenium import webdriver from selenium.common.exceptions import TimeoutException +from selenium.webdriver.common.proxy import Proxy, ProxyType from loguru import logger from selenium.webdriver.common.by import By import time class Webdriver: - def __init__(self, width: int, height: int, timeout_seconds: int, facebook_accept_cookies: bool = False) -> webdriver: + def __init__(self, width: int, height: int, timeout_seconds: int, facebook_accept_cookies: bool = False, http_proxy: str = "") -> webdriver: self.width = width self.height = height self.timeout_seconds = timeout_seconds self.facebook_accept_cookies = facebook_accept_cookies + self.http_proxy = http_proxy def __enter__(self) -> webdriver: options = webdriver.FirefoxOptions() options.add_argument("--headless") + options.add_argument(f'--proxy-server={self.http_proxy}') options.set_preference('network.protocol-handler.external.tg', False) try: self.driver = webdriver.Firefox(options=options) diff --git a/src/auto_archiver/version.py b/src/auto_archiver/version.py index 552a58a..2cc14f6 100644 --- a/src/auto_archiver/version.py +++ b/src/auto_archiver/version.py @@ -1,9 +1,9 @@ _MAJOR = "0" -_MINOR = "8" +_MINOR = "9" # On main and in a nightly release the patch should be one ahead of the last # released build. -_PATCH = "1" +_PATCH = "0" # This is mainly for nightly builds which have the suffix ".dev$DATE". See # https://semver.org/#is-v123-a-semantic-version for the semantics. _SUFFIX = ""