kopia lustrzana https://github.com/bellingcat/auto-archiver
Merge branch 'main' into feat/yt-dlp-pots
erinhmclark
commit
7872d9356c
|
|
@ -20,8 +20,7 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12"]
|
||||
os: [ubuntu-22.04]
|
||||
#TODO: re-enable ubuntu-latest, this is disabled as oscrypto cannot be pinned to github commit and pushed to pypi
|
||||
os: [ubuntu-22.04, ubuntu-latest]
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ./
|
||||
|
|
|
|||
|
|
@ -1,15 +1,23 @@
|
|||
name: Download Tests
|
||||
|
||||
on:
|
||||
# Schedule this workflow to run every Monday at 14:35 UTC
|
||||
schedule:
|
||||
- cron: '35 14 * * 1'
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
|
||||
# For PRs, run the tests-core workflow first
|
||||
workflow_run:
|
||||
workflows: ["Core Tests"]
|
||||
types:
|
||||
- completed
|
||||
branches: [main]
|
||||
paths:
|
||||
- src/**
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
# Only run if the triggering workflow (tests-core) succeeded
|
||||
if: ${{ github.event_name == 'schedule' || github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
|
|
@ -21,6 +29,8 @@ jobs:
|
|||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_branch || github.ref }}
|
||||
|
||||
- name: Install poetry
|
||||
run: pipx install poetry
|
||||
|
|
|
|||
|
|
@ -54,18 +54,6 @@ doc = ["Sphinx (>=8.2,<9.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)",
|
|||
test = ["anyio[trio]", "blockbuster (>=1.5.23)", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1) ; python_version >= \"3.10\"", "uvloop (>=0.21) ; platform_python_implementation == \"CPython\" and platform_system != \"Windows\" and python_version < \"3.14\""]
|
||||
trio = ["trio (>=0.26.1)"]
|
||||
|
||||
[[package]]
|
||||
name = "asn1crypto"
|
||||
version = "1.5.1"
|
||||
description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"},
|
||||
{file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "astroid"
|
||||
version = "3.3.9"
|
||||
|
|
@ -385,22 +373,6 @@ files = [
|
|||
{file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "certvalidator"
|
||||
version = "0.11.1"
|
||||
description = "Validates X.509 certificates and paths"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "certvalidator-0.11.1-py2.py3-none-any.whl", hash = "sha256:77520b269f516d4fb0902998d5bd0eb3727fe153b659aa1cb828dcf12ea6b8de"},
|
||||
{file = "certvalidator-0.11.1.tar.gz", hash = "sha256:922d141c94393ab285ca34338e18dd4093e3ae330b1f278e96c837cb62cffaad"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
asn1crypto = ">=0.18.1"
|
||||
oscrypto = ">=0.16.1"
|
||||
|
||||
[[package]]
|
||||
name = "cffi"
|
||||
version = "1.17.1"
|
||||
|
|
@ -408,6 +380,7 @@ description = "Foreign Function Interface for Python calling C code."
|
|||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main"]
|
||||
markers = "os_name == \"nt\" and implementation_name != \"pypy\" or platform_python_implementation != \"PyPy\""
|
||||
files = [
|
||||
{file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"},
|
||||
{file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"},
|
||||
|
|
@ -625,48 +598,60 @@ markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\"",
|
|||
|
||||
[[package]]
|
||||
name = "cryptography"
|
||||
version = "41.0.7"
|
||||
version = "44.0.2"
|
||||
description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
python-versions = "!=3.9.0,!=3.9.1,>=3.7"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"},
|
||||
{file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"},
|
||||
{file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"},
|
||||
{file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"},
|
||||
{file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"},
|
||||
{file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"},
|
||||
{file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"},
|
||||
{file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"},
|
||||
{file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"},
|
||||
{file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"},
|
||||
{file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"},
|
||||
{file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"},
|
||||
{file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"},
|
||||
{file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"},
|
||||
{file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"},
|
||||
{file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"},
|
||||
{file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"},
|
||||
{file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"},
|
||||
{file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"},
|
||||
{file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"},
|
||||
{file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"},
|
||||
{file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"},
|
||||
{file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"},
|
||||
{file = "cryptography-44.0.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:efcfe97d1b3c79e486554efddeb8f6f53a4cdd4cf6086642784fa31fc384e1d7"},
|
||||
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29ecec49f3ba3f3849362854b7253a9f59799e3763b0c9d0826259a88efa02f1"},
|
||||
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc821e161ae88bfe8088d11bb39caf2916562e0a2dc7b6d56714a48b784ef0bb"},
|
||||
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3c00b6b757b32ce0f62c574b78b939afab9eecaf597c4d624caca4f9e71e7843"},
|
||||
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7bdcd82189759aba3816d1f729ce42ffded1ac304c151d0a8e89b9996ab863d5"},
|
||||
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4973da6ca3db4405c54cd0b26d328be54c7747e89e284fcff166132eb7bccc9c"},
|
||||
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4e389622b6927d8133f314949a9812972711a111d577a5d1f4bee5e58736b80a"},
|
||||
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f514ef4cd14bb6fb484b4a60203e912cfcb64f2ab139e88c2274511514bf7308"},
|
||||
{file = "cryptography-44.0.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1bc312dfb7a6e5d66082c87c34c8a62176e684b6fe3d90fcfe1568de675e6688"},
|
||||
{file = "cryptography-44.0.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b721b8b4d948b218c88cb8c45a01793483821e709afe5f622861fc6182b20a7"},
|
||||
{file = "cryptography-44.0.2-cp37-abi3-win32.whl", hash = "sha256:51e4de3af4ec3899d6d178a8c005226491c27c4ba84101bfb59c901e10ca9f79"},
|
||||
{file = "cryptography-44.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:c505d61b6176aaf982c5717ce04e87da5abc9a36a5b39ac03905c4aafe8de7aa"},
|
||||
{file = "cryptography-44.0.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:8e0ddd63e6bf1161800592c71ac794d3fb8001f2caebe0966e77c5234fa9efc3"},
|
||||
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81276f0ea79a208d961c433a947029e1a15948966658cf6710bbabb60fcc2639"},
|
||||
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a1e657c0f4ea2a23304ee3f964db058c9e9e635cc7019c4aa21c330755ef6fd"},
|
||||
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6210c05941994290f3f7f175a4a57dbbb2afd9273657614c506d5976db061181"},
|
||||
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d1c3572526997b36f245a96a2b1713bf79ce99b271bbcf084beb6b9b075f29ea"},
|
||||
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b042d2a275c8cee83a4b7ae30c45a15e6a4baa65a179a0ec2d78ebb90e4f6699"},
|
||||
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:d03806036b4f89e3b13b6218fefea8d5312e450935b1a2d55f0524e2ed7c59d9"},
|
||||
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c7362add18b416b69d58c910caa217f980c5ef39b23a38a0880dfd87bdf8cd23"},
|
||||
{file = "cryptography-44.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8cadc6e3b5a1f144a039ea08a0bdb03a2a92e19c46be3285123d32029f40a922"},
|
||||
{file = "cryptography-44.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6f101b1f780f7fc613d040ca4bdf835c6ef3b00e9bd7125a4255ec574c7916e4"},
|
||||
{file = "cryptography-44.0.2-cp39-abi3-win32.whl", hash = "sha256:3dc62975e31617badc19a906481deacdeb80b4bb454394b4098e3f2525a488c5"},
|
||||
{file = "cryptography-44.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:5f6f90b72d8ccadb9c6e311c775c8305381db88374c65fa1a68250aa8a9cb3a6"},
|
||||
{file = "cryptography-44.0.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:af4ff3e388f2fa7bff9f7f2b31b87d5651c45731d3e8cfa0944be43dff5cfbdb"},
|
||||
{file = "cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0529b1d5a0105dd3731fa65680b45ce49da4d8115ea76e9da77a875396727b41"},
|
||||
{file = "cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7ca25849404be2f8e4b3c59483d9d3c51298a22c1c61a0e84415104dacaf5562"},
|
||||
{file = "cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:268e4e9b177c76d569e8a145a6939eca9a5fec658c932348598818acf31ae9a5"},
|
||||
{file = "cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:9eb9d22b0a5d8fd9925a7764a054dca914000607dff201a24c791ff5c799e1fa"},
|
||||
{file = "cryptography-44.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2bf7bf75f7df9715f810d1b038870309342bff3069c5bd8c6b96128cb158668d"},
|
||||
{file = "cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:909c97ab43a9c0c0b0ada7a1281430e4e5ec0458e6d9244c0e821bbf152f061d"},
|
||||
{file = "cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:96e7a5e9d6e71f9f4fca8eebfd603f8e86c5225bb18eb621b2c1e50b290a9471"},
|
||||
{file = "cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d1b3031093a366ac767b3feb8bcddb596671b3aaff82d4050f984da0c248b615"},
|
||||
{file = "cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:04abd71114848aa25edb28e225ab5f268096f44cf0127f3d36975bdf1bdf3390"},
|
||||
{file = "cryptography-44.0.2.tar.gz", hash = "sha256:c63454aa261a0cf0c5b4718349629793e9e634993538db841165b3df74f37ec0"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
cffi = ">=1.12"
|
||||
cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""}
|
||||
|
||||
[package.extras]
|
||||
docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"]
|
||||
docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"]
|
||||
nox = ["nox"]
|
||||
pep8test = ["black", "check-sdist", "mypy", "ruff"]
|
||||
sdist = ["build"]
|
||||
docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0) ; python_version >= \"3.8\""]
|
||||
docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"]
|
||||
nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2) ; python_version >= \"3.8\""]
|
||||
pep8test = ["check-sdist ; python_version >= \"3.8\"", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"]
|
||||
sdist = ["build (>=1.0.0)"]
|
||||
ssh = ["bcrypt (>=3.1.5)"]
|
||||
test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"]
|
||||
test = ["certifi (>=2024)", "cryptography-vectors (==44.0.2)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"]
|
||||
test-randomorder = ["pytest-randomly"]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1445,21 +1430,6 @@ files = [
|
|||
pycryptodomex = ">=3.3.1"
|
||||
python-bitcoinlib = ">=0.9.0,<0.13.0"
|
||||
|
||||
[[package]]
|
||||
name = "oscrypto"
|
||||
version = "1.3.0"
|
||||
description = "TLS (SSL) sockets, key generation, encryption, decryption, signing, verification and KDFs using the OS crypto libraries. Does not require a compiler, and relies on the OS for patching. Works on Windows, OS X and Linux/BSD."
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "oscrypto-1.3.0-py2.py3-none-any.whl", hash = "sha256:2b2f1d2d42ec152ca90ccb5682f3e051fb55986e1b170ebde472b133713e7085"},
|
||||
{file = "oscrypto-1.3.0.tar.gz", hash = "sha256:6f5fef59cb5b3708321db7cca56aed8ad7e662853351e7991fcf60ec606d47a4"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
asn1crypto = ">=1.5.1"
|
||||
|
||||
[[package]]
|
||||
name = "outcome"
|
||||
version = "1.3.0.post0"
|
||||
|
|
@ -1743,6 +1713,7 @@ description = "C parser in Python"
|
|||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main"]
|
||||
markers = "os_name == \"nt\" and implementation_name != \"pypy\" or platform_python_implementation != \"PyPy\""
|
||||
files = [
|
||||
{file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"},
|
||||
{file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},
|
||||
|
|
@ -1831,25 +1802,6 @@ files = [
|
|||
[package.extras]
|
||||
windows-terminal = ["colorama (>=0.4.6)"]
|
||||
|
||||
[[package]]
|
||||
name = "pyopenssl"
|
||||
version = "24.2.1"
|
||||
description = "Python wrapper module around the OpenSSL library"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "pyOpenSSL-24.2.1-py3-none-any.whl", hash = "sha256:967d5719b12b243588573f39b0c677637145c7a1ffedcd495a487e58177fbb8d"},
|
||||
{file = "pyopenssl-24.2.1.tar.gz", hash = "sha256:4247f0dbe3748d560dcbb2ff3ea01af0f9a1a001ef5f7c4c647956ed8cbf0e95"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
cryptography = ">=41.0.5,<44"
|
||||
|
||||
[package.extras]
|
||||
docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx-rtd-theme"]
|
||||
test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"]
|
||||
|
||||
[[package]]
|
||||
name = "pyparsing"
|
||||
version = "3.2.2"
|
||||
|
|
@ -2247,6 +2199,37 @@ files = [
|
|||
[package.dependencies]
|
||||
six = ">=1.7.0"
|
||||
|
||||
[[package]]
|
||||
name = "rfc3161-client"
|
||||
version = "1.0.1"
|
||||
description = ""
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "rfc3161_client-1.0.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:75d8c9d255fa79b9ae4aa27cee519893599efd79f9e6c24a1194dd296ce1c210"},
|
||||
{file = "rfc3161_client-1.0.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0d3db059fe08d8b6b06aff89e133fcc352ffea1a1dafadb116dda9dae59d0689"},
|
||||
{file = "rfc3161_client-1.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdef0c9d3213ca5b79d7f76ada48ae10c5011cb25abed2f6df07b344d16d1c28"},
|
||||
{file = "rfc3161_client-1.0.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7c34ce4d7d2bf5207c54de3a771e757f1f8bb04a8469d3cef6aefe074841064d"},
|
||||
{file = "rfc3161_client-1.0.1-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4809f2fcfb5f8b42261a7b831929f62a297b584c8d1f4d242eae5e9447674b6"},
|
||||
{file = "rfc3161_client-1.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a644b220b7f0f0be7856f49b043651982bd76e7aa9eb17b3e4e303fde36ed5a1"},
|
||||
{file = "rfc3161_client-1.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bb03a5a77b07adf766b7daac6cb8b7a8337ffc8f6d6046af74469973f52df8e1"},
|
||||
{file = "rfc3161_client-1.0.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:d6c6e4626780b1c531d32d6a126d6c27865b1eb59c65e8b0f1f8f94aa3205285"},
|
||||
{file = "rfc3161_client-1.0.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:912c2f049ce23d0f1c173b6fbd8673f964a27ad97907064dbc74f86dd0d95d15"},
|
||||
{file = "rfc3161_client-1.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:081211a1b602b6dff7feb314d39ca2229c8db4e8cf55eef0c35b460470f4b2bb"},
|
||||
{file = "rfc3161_client-1.0.1-cp39-abi3-win32.whl", hash = "sha256:59efa8fddf72a15e397276fe512dbfb99c0dc95032b495815bfc4f8f16302f2c"},
|
||||
{file = "rfc3161_client-1.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:5381a63d5ed5b3c257cb18aacf3f737b1a1ad6df634290fe689b6d601c61cd24"},
|
||||
{file = "rfc3161_client-1.0.1.tar.gz", hash = "sha256:1c951f3912b90c6d3f3505e644b74ee08543387253647b86459addbffb16f63f"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
cryptography = ">=43,<45"
|
||||
|
||||
[package.extras]
|
||||
dev = ["maturin (>=1.7,<2.0)", "rfc3161-client[doc,lint,test]"]
|
||||
lint = ["interrogate", "ruff (>=0.7,<0.12)"]
|
||||
test = ["coverage[toml]", "pretend", "pytest", "pytest-cov"]
|
||||
|
||||
[[package]]
|
||||
name = "rich"
|
||||
version = "13.9.4"
|
||||
|
|
@ -2946,26 +2929,6 @@ outcome = ">=1.2.0"
|
|||
trio = ">=0.11"
|
||||
wsproto = ">=0.14"
|
||||
|
||||
[[package]]
|
||||
name = "tsp-client"
|
||||
version = "0.2.0"
|
||||
description = "An IETF Time-Stamp Protocol (TSP) (RFC 3161) client"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "tsp-client-0.2.0.tar.gz", hash = "sha256:6e66148dd116322eb44a7484e5ad33bbe640b997343c443de9cc70fc5eb19987"},
|
||||
{file = "tsp_client-0.2.0-py3-none-any.whl", hash = "sha256:0b790d10a68d66782c13f1d7cc7f5206df26b49826c1da80944b7c05b1731784"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
asn1crypto = ">=0.24.0"
|
||||
pyOpenSSL = ">=20.0.0"
|
||||
requests = ">=2.18.4"
|
||||
|
||||
[package.extras]
|
||||
tests = ["build", "coverage", "mypy", "ruff", "wheel"]
|
||||
|
||||
[[package]]
|
||||
name = "typing-extensions"
|
||||
version = "4.12.2"
|
||||
|
|
@ -3416,4 +3379,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"]
|
|||
[metadata]
|
||||
lock-version = "2.1"
|
||||
python-versions = ">=3.10,<3.13"
|
||||
content-hash = "147f2ae2e72b23cc47b8b6e398de52fd327b27851b3a23e037bbe2f9c9c90755"
|
||||
content-hash = "ac5d473189adbadb3ee5d8a36e1898a39725755704e0677768303ae46bc246c8"
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|||
|
||||
[project]
|
||||
name = "auto-archiver"
|
||||
version = "0.13.8"
|
||||
version = "0.13.9"
|
||||
description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
|
||||
|
||||
requires-python = ">=3.10,<3.13"
|
||||
|
|
@ -41,8 +41,6 @@ dependencies = [
|
|||
"instaloader (>=0.0.0)",
|
||||
"tqdm (>=0.0.0)",
|
||||
"jinja2 (>=0.0.0)",
|
||||
"pyOpenSSL (==24.2.1)",
|
||||
"cryptography (>=41.0.0,<42.0.0)",
|
||||
"boto3 (>=1.28.0,<2.0.0)",
|
||||
"dataclasses-json (>=0.0.0)",
|
||||
"yt-dlp (>=2025.3.21,<2026.0.0)",
|
||||
|
|
@ -53,10 +51,10 @@ dependencies = [
|
|||
"jsonlines (>=0.0.0)",
|
||||
"pysubs2 (>=0.0.0)",
|
||||
"retrying (>=0.0.0)",
|
||||
"tsp-client (>=0.0.0)",
|
||||
"certvalidator (>=0.0.0)",
|
||||
"rich-argparse (>=1.6.0,<2.0.0)",
|
||||
"ruamel-yaml (>=0.18.10,<0.19.0)",
|
||||
"rfc3161-client (>=1.0.1,<2.0.0)",
|
||||
"cryptography (>44.0.1,<45.0.0)",
|
||||
"opentimestamps (>=0.4.5,<0.5.0)",
|
||||
"bgutil-ytdlp-pot-provider (>=0.7.3,<0.8.0)",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -214,11 +214,8 @@ class LazyBaseModule:
|
|||
|
||||
# check external dependencies are installed
|
||||
def check_deps(deps, check):
|
||||
for dep in deps:
|
||||
if not len(dep):
|
||||
# clear out any empty strings that a user may have erroneously added
|
||||
continue
|
||||
if not check(dep):
|
||||
for dep in filter(lambda d: len(d.strip()), deps):
|
||||
if not check(dep.strip()):
|
||||
logger.error(
|
||||
f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. \
|
||||
Have you installed the required dependencies for the '{self.name}' module? See the documentation for more information."
|
||||
|
|
|
|||
|
|
@ -3,30 +3,38 @@
|
|||
"type": ["enricher"],
|
||||
"requires_setup": True,
|
||||
"dependencies": {
|
||||
"python": ["loguru", "slugify", "tsp_client", "asn1crypto", "certvalidator", "certifi"],
|
||||
"python": ["loguru", "slugify", "cryptography", "rfc3161_client", "certifi"],
|
||||
},
|
||||
"configs": {
|
||||
"tsa_urls": {
|
||||
"default": [
|
||||
# [Adobe Approved Trust List] and [Windows Cert Store]
|
||||
"http://timestamp.digicert.com",
|
||||
# See https://github.com/trailofbits/rfc3161-client/issues/46 for a list of valid TSAs
|
||||
# Full list of TSAs: https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710
|
||||
"http://timestamp.identrust.com",
|
||||
# "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping
|
||||
# "https://timestamp.sectigo.com", # wait 15 seconds between each request.
|
||||
# [Adobe: European Union Trusted Lists].
|
||||
# "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request.
|
||||
# [Windows Cert Store]
|
||||
"http://timestamp.globalsign.com/tsa/r6advanced1",
|
||||
# [Adobe: European Union Trusted Lists] and [Windows Cert Store]
|
||||
# "http://ts.quovadisglobal.com/eu", # not valid for timestamping
|
||||
# "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain
|
||||
# "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain
|
||||
# "http://tsa.sep.bg", # self-signed certificate in certificate chain
|
||||
# "http://tsa.izenpe.com", #unable to get local issuer certificate
|
||||
# "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate
|
||||
"http://tss.accv.es:8318/tsa",
|
||||
"http://timestamp.ssl.trustwave.com",
|
||||
"http://zeitstempel.dfn.de",
|
||||
"http://ts.ssl.com",
|
||||
# "http://tsa.izenpe.com", # self-signed
|
||||
"http://tsa.lex-persona.com/tsa",
|
||||
# "http://ca.signfiles.com/TSAServer.aspx", # self-signed
|
||||
# "http://tsa.sinpe.fi.cr/tsaHttp/", # self-signed
|
||||
# "http://tsa.cra.ge/signserver/tsa?workerName=qtsa", # self-signed
|
||||
"http://tss.cnbs.gob.hn/TSS/HttpTspServer",
|
||||
"http://dss.nowina.lu/pki-factory/tsa/good-tsa",
|
||||
# "https://freetsa.org/tsr", # self-signed
|
||||
],
|
||||
"help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.",
|
||||
},
|
||||
"cert_authorities": {
|
||||
"default": None,
|
||||
"help": "Path to a file containing trusted Certificate Authorities (CAs) in PEM format. If empty, the default system authorities are used.",
|
||||
"type": "str",
|
||||
},
|
||||
"allow_selfsigned": {
|
||||
"default": False,
|
||||
"help": "Whether or not to allow and save self-signed Timestamping certificates. This allows for a greater range of timestamping servers to be used, \
|
||||
but they are not trusted authorities",
|
||||
"type": "bool"
|
||||
}
|
||||
},
|
||||
"description": """
|
||||
|
|
|
|||
|
|
@ -1,15 +1,22 @@
|
|||
import os
|
||||
from loguru import logger
|
||||
from tsp_client import TSPSigner, SigningSettings, TSPVerifier
|
||||
from tsp_client.algorithms import DigestAlgorithm
|
||||
from importlib.metadata import version
|
||||
from asn1crypto.cms import ContentInfo
|
||||
from certvalidator import CertificateValidator, ValidationContext
|
||||
from asn1crypto import pem
|
||||
import certifi
|
||||
|
||||
from importlib.metadata import version
|
||||
import hashlib
|
||||
|
||||
from slugify import slugify
|
||||
import requests
|
||||
from loguru import logger
|
||||
|
||||
from rfc3161_client import (decode_timestamp_response,TimestampRequestBuilder,TimeStampResponse, VerifierBuilder)
|
||||
from rfc3161_client import VerificationError as Rfc3161VerificationError
|
||||
from rfc3161_client.base import HashAlgorithm
|
||||
from rfc3161_client.tsp import SignedData
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
import certifi
|
||||
from auto_archiver.core import Enricher
|
||||
from auto_archiver.core import Metadata, Media
|
||||
from auto_archiver.version import __version__
|
||||
|
||||
|
||||
class TimestampingEnricher(Enricher):
|
||||
|
|
@ -21,6 +28,25 @@ class TimestampingEnricher(Enricher):
|
|||
See https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710 for list of timestamp authorities.
|
||||
"""
|
||||
|
||||
session = None
|
||||
|
||||
def setup(self):
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(
|
||||
{
|
||||
"Content-Type": "application/timestamp-query",
|
||||
"User-Agent": f"Auto-Archiver {__version__}",
|
||||
"Accept": "application/timestamp-reply",
|
||||
}
|
||||
)
|
||||
|
||||
def cleaup(self) -> None:
|
||||
"""
|
||||
Terminates the underlying network session.
|
||||
"""
|
||||
if self.session:
|
||||
self.session.close()
|
||||
|
||||
def enrich(self, to_enrich: Metadata) -> None:
|
||||
url = to_enrich.get_url()
|
||||
logger.debug(f"RFC3161 timestamping existing files for {url=}")
|
||||
|
|
@ -34,8 +60,8 @@ class TimestampingEnricher(Enricher):
|
|||
logger.warning(f"No hashes found in {url=}")
|
||||
return
|
||||
|
||||
tmp_dir = self.tmp_dir
|
||||
hashes_fn = os.path.join(tmp_dir, "hashes.txt")
|
||||
|
||||
hashes_fn = os.path.join(self.tmp_dir, "hashes.txt")
|
||||
|
||||
data_to_sign = "\n".join(hashes)
|
||||
with open(hashes_fn, "w") as f:
|
||||
|
|
@ -43,62 +69,160 @@ class TimestampingEnricher(Enricher):
|
|||
hashes_media = Media(filename=hashes_fn)
|
||||
|
||||
timestamp_tokens = []
|
||||
from slugify import slugify
|
||||
|
||||
for tsa_url in self.tsa_urls:
|
||||
try:
|
||||
signing_settings = SigningSettings(tsp_server=tsa_url, digest_algorithm=DigestAlgorithm.SHA256)
|
||||
signer = TSPSigner()
|
||||
message = bytes(data_to_sign, encoding="utf8")
|
||||
# send TSQ and get TSR from the TSA server
|
||||
signed = signer.sign(message=message, signing_settings=signing_settings)
|
||||
# fail if there's any issue with the certificates, uses certifi list of trusted CAs
|
||||
TSPVerifier(certifi.where()).verify(signed, message=message)
|
||||
# download and verify timestamping certificate
|
||||
cert_chain = self.download_and_verify_certificate(signed)
|
||||
# continue with saving the timestamp token
|
||||
tst_fn = os.path.join(tmp_dir, f"timestamp_token_{slugify(tsa_url)}")
|
||||
with open(tst_fn, "wb") as f:
|
||||
f.write(signed)
|
||||
timestamp_tokens.append(Media(filename=tst_fn).set("tsa", tsa_url).set("cert_chain", cert_chain))
|
||||
message = bytes(data_to_sign, encoding='utf8')
|
||||
|
||||
logger.debug(f"Timestamping {url=} with {tsa_url=}")
|
||||
signed: TimeStampResponse = self.sign_data(tsa_url, message)
|
||||
|
||||
# fail if there's any issue with the certificates, uses certifi list of trusted CAs or the user-defined `cert_authorities`
|
||||
root_cert = self.verify_signed(signed, message)
|
||||
|
||||
if not root_cert:
|
||||
if self.allow_selfsigned:
|
||||
logger.warning(f"Allowing self-signed certificat from TSA {tsa_url=}")
|
||||
else:
|
||||
raise ValueError(f"No valid root certificate found for {tsa_url=}. Are you sure it's a trusted TSA? Or define an alternative trusted root with `cert_authorities`. (tried: {self.cert_authorities or certifi.where()})")
|
||||
|
||||
# save the timestamping certificate
|
||||
cert_chain = self.save_certificate(signed, root_cert)
|
||||
|
||||
timestamp_token_path = self.save_timestamp_token(signed.time_stamp_token(), tsa_url)
|
||||
timestamp_tokens.append(Media(filename=timestamp_token_path).set("tsa", tsa_url).set("cert_chain", cert_chain))
|
||||
except Exception as e:
|
||||
logger.warning(f"Error while timestamping {url=} with {tsa_url=}: {e}")
|
||||
|
||||
if len(timestamp_tokens):
|
||||
hashes_media.set("timestamp_authority_files", timestamp_tokens)
|
||||
hashes_media.set("certifi v", version("certifi"))
|
||||
hashes_media.set("tsp_client v", version("tsp_client"))
|
||||
hashes_media.set("certvalidator v", version("certvalidator"))
|
||||
hashes_media.set("rfc3161-client v", version("rfc3161_client"))
|
||||
hashes_media.set("cryptography v", version("cryptography"))
|
||||
to_enrich.add_media(hashes_media, id="timestamped_hashes")
|
||||
to_enrich.set("timestamped", True)
|
||||
logger.success(f"{len(timestamp_tokens)} timestamp tokens created for {url=}")
|
||||
else:
|
||||
logger.warning(f"No successful timestamps for {url=}")
|
||||
|
||||
def download_and_verify_certificate(self, signed: bytes) -> list[Media]:
|
||||
def save_timestamp_token(self, timestamp_token: bytes, tsa_url: str) -> str:
|
||||
"""
|
||||
Takes a timestamp token, and saves it to a file with the TSA URL as part of the filename.
|
||||
"""
|
||||
tst_path = os.path.join(self.tmp_dir, f"timestamp_token_{slugify(tsa_url)}")
|
||||
with open(tst_path, "wb") as f:
|
||||
f.write(timestamp_token)
|
||||
return tst_path
|
||||
|
||||
def verify_signed(self, timestamp_response: TimeStampResponse, message: bytes) -> x509.Certificate:
|
||||
"""
|
||||
Verify a Signed Timestamp Response is trusted by a known Certificate Authority.
|
||||
|
||||
Args:
|
||||
timestamp_response (TimeStampResponse): The signed timestamp response.
|
||||
message (bytes): The message that was timestamped.
|
||||
|
||||
Returns:
|
||||
x509.Certificate: A valid root certificate that was used to sign the timestamp response, or None
|
||||
|
||||
Raises:
|
||||
ValueError: If no valid root certificate was found in the trusted root store.
|
||||
"""
|
||||
|
||||
trusted_root_path = self.cert_authorities or certifi.where()
|
||||
cert_authorities = []
|
||||
|
||||
with open(trusted_root_path, 'rb') as f:
|
||||
cert_authorities = x509.load_pem_x509_certificates(f.read())
|
||||
|
||||
if not cert_authorities:
|
||||
raise ValueError(f"No trusted roots found in {trusted_root_path}.")
|
||||
|
||||
timestamp_certs = self.tst_certs(timestamp_response)
|
||||
intermediate_certs = timestamp_certs[1:-1]
|
||||
|
||||
message_hash = None
|
||||
hash_algorithm = timestamp_response.tst_info.message_imprint.hash_algorithm
|
||||
if hash_algorithm == x509.ObjectIdentifier(value="2.16.840.1.101.3.4.2.3"):
|
||||
message_hash = hashlib.sha512(message).digest()
|
||||
elif hash_algorithm == x509.ObjectIdentifier(value="2.16.840.1.101.3.4.2.1"):
|
||||
message_hash = hashlib.sha256(message).digest()
|
||||
else:
|
||||
raise ValueError(f"Unsupported hash algorithm: {hash_algorithm}")
|
||||
|
||||
for certificate in cert_authorities:
|
||||
builder = VerifierBuilder()
|
||||
builder.add_root_certificate(certificate)
|
||||
|
||||
for intermediate_cert in intermediate_certs:
|
||||
builder.add_intermediate_certificate(intermediate_cert)
|
||||
|
||||
verifier = builder.build()
|
||||
|
||||
|
||||
try:
|
||||
verifier.verify(timestamp_response, message_hash)
|
||||
return certificate
|
||||
except Rfc3161VerificationError:
|
||||
continue
|
||||
|
||||
return None
|
||||
|
||||
def sign_data(self, tsa_url: str, bytes_data: bytes) -> TimeStampResponse:
|
||||
# see https://github.com/sigstore/sigstore-python/blob/99948d5b80525a5a104e904ffea58169dc6e0629/sigstore/_internal/timestamp.py#L84-L121
|
||||
|
||||
timestamp_request = (
|
||||
TimestampRequestBuilder().data(bytes_data).nonce(nonce=True).build()
|
||||
)
|
||||
try:
|
||||
response = self.session.post(tsa_url, data=timestamp_request.as_bytes(), timeout=10)
|
||||
response.raise_for_status()
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"Error while sending request to {tsa_url=}: {e}")
|
||||
raise
|
||||
|
||||
# Check that we can parse the response but do not *verify* it
|
||||
try:
|
||||
timestamp_response = decode_timestamp_response(response.content)
|
||||
except ValueError as e:
|
||||
logger.error(f"Invalid timestamp response from server {tsa_url}: {e}")
|
||||
raise
|
||||
return timestamp_response
|
||||
|
||||
def tst_certs(self, tsp_response: TimeStampResponse):
|
||||
signed_data: SignedData = tsp_response.signed_data
|
||||
certs = [x509.load_der_x509_certificate(c) for c in signed_data.certificates]
|
||||
# reorder the certs to be in the correct order
|
||||
ordered_certs = []
|
||||
if len(certs) == 1:
|
||||
return certs
|
||||
|
||||
while(len(ordered_certs) < len(certs)):
|
||||
if len(ordered_certs) == 0:
|
||||
for cert in certs:
|
||||
if not [c for c in certs if cert.subject == c.issuer]:
|
||||
ordered_certs.append(cert)
|
||||
break
|
||||
else:
|
||||
for cert in certs:
|
||||
if cert.subject == ordered_certs[-1].issuer:
|
||||
ordered_certs.append(cert)
|
||||
break
|
||||
return ordered_certs
|
||||
|
||||
def save_certificate(self, tsp_response: TimeStampResponse, verified_root_cert: x509.Certificate) -> list[Media]:
|
||||
# returns the leaf certificate URL, fails if not set
|
||||
tst = ContentInfo.load(signed)
|
||||
|
||||
trust_roots = []
|
||||
with open(certifi.where(), "rb") as f:
|
||||
for _, _, der_bytes in pem.unarmor(f.read(), multiple=True):
|
||||
trust_roots.append(der_bytes)
|
||||
context = ValidationContext(trust_roots=trust_roots)
|
||||
certificates = self.tst_certs(tsp_response)
|
||||
|
||||
certificates = tst["content"]["certificates"]
|
||||
first_cert = certificates[0].dump()
|
||||
intermediate_certs = []
|
||||
for i in range(1, len(certificates)): # cannot use list comprehension [1:]
|
||||
intermediate_certs.append(certificates[i].dump())
|
||||
|
||||
validator = CertificateValidator(first_cert, intermediate_certs=intermediate_certs, validation_context=context)
|
||||
path = validator.validate_usage({"digital_signature"}, extended_key_usage={"time_stamping"})
|
||||
if verified_root_cert:
|
||||
# add the verified root certificate (if there is one - self signed certs will have None here)
|
||||
certificates += [verified_root_cert]
|
||||
|
||||
cert_chain = []
|
||||
for cert in path:
|
||||
cert_fn = os.path.join(self.tmp_dir, f"{str(cert.serial_number)[:20]}.crt")
|
||||
for i, cert in enumerate(certificates):
|
||||
cert_fn = os.path.join(self.tmp_dir, f"{i+1} – {str(cert.serial_number)[:20]}.crt")
|
||||
with open(cert_fn, "wb") as f:
|
||||
f.write(cert.dump())
|
||||
cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.native["common_name"]))
|
||||
f.write(cert.public_bytes(encoding=serialization.Encoding.PEM))
|
||||
cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME)[0].value))
|
||||
|
||||
return cert_chain
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from typing import Dict, Tuple
|
|||
import hashlib
|
||||
|
||||
import pytest
|
||||
from auto_archiver.core.metadata import Metadata
|
||||
from auto_archiver.core.metadata import Metadata, Media
|
||||
from auto_archiver.core.module import ModuleFactory
|
||||
|
||||
# Test names inserted into this list will be run last. This is useful for expensive/costly tests
|
||||
|
|
@ -140,6 +140,14 @@ def mock_binary_dependencies(mocker):
|
|||
return mock_shutil_which
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_media(tmp_path) -> Media:
|
||||
"""Fixture creating a Media object with temporary source file"""
|
||||
src_file = tmp_path / "source.txt"
|
||||
src_file.write_text("test content")
|
||||
return Media(_key="subdir/test.txt", filename=str(src_file))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_datetime():
|
||||
return datetime(2023, 1, 1, 12, 0, tzinfo=timezone.utc)
|
||||
|
|
|
|||
Plik binarny nie jest wyświetlany.
Plik binarny nie jest wyświetlany.
Plik binarny nie jest wyświetlany.
Plik binarny nie jest wyświetlany.
Plik binarny nie jest wyświetlany.
|
|
@ -0,0 +1,215 @@
|
|||
from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from rfc3161_client import (
|
||||
TimeStampResponse,
|
||||
decode_timestamp_response,
|
||||
)
|
||||
import requests
|
||||
|
||||
from auto_archiver.modules.timestamping_enricher.timestamping_enricher import TimestampingEnricher
|
||||
from auto_archiver.core import Metadata
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def timestamp_response() -> TimeStampResponse:
|
||||
with open("tests/data/timestamping/valid_timestamp.tsr", "rb") as f:
|
||||
return decode_timestamp_response(f.read())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wrong_order_timestamp_response() -> TimeStampResponse:
|
||||
with open("tests/data/timestamping/rfc3161-client-issue-104.tsr", "rb") as f:
|
||||
return decode_timestamp_response(f.read())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def selfsigned_response() -> TimeStampResponse:
|
||||
with open("tests/data/timestamping/self_signed.tsr", "rb") as f:
|
||||
return decode_timestamp_response(f.read())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def digicert_response() -> TimeStampResponse:
|
||||
with open("tests/data/timestamping/digicert.tsr", "rb") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def filehash():
|
||||
return "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef"
|
||||
|
||||
|
||||
@pytest.mark.download
|
||||
def test_enriching(setup_module, sample_media):
|
||||
tsp: TimestampingEnricher = setup_module("timestamping_enricher")
|
||||
|
||||
# tests the current TSAs set as default in the __manifest__ to make sure they are all still working
|
||||
|
||||
# test the enrich method
|
||||
metadata = Metadata().set_url("https://example.com")
|
||||
sample_media.set("hash", "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef")
|
||||
metadata.add_media(sample_media)
|
||||
tsp.enrich(metadata)
|
||||
|
||||
|
||||
def test_full_enriching_selfsigned(setup_module, sample_media, mocker, selfsigned_response, filehash):
|
||||
mock_post = mocker.patch("requests.sessions.Session.post")
|
||||
mock_post.return_value.status_code = 200
|
||||
mock_decode_timestamp_response = mocker.patch(
|
||||
"auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response"
|
||||
)
|
||||
mock_decode_timestamp_response.return_value = selfsigned_response
|
||||
|
||||
tsp: TimestampingEnricher = setup_module("timestamping_enricher", {"tsa_urls": ["http://timestamp.identrust.com"]})
|
||||
metadata = Metadata().set_url("https://example.com")
|
||||
sample_media.set("hash", filehash)
|
||||
metadata.add_media(sample_media)
|
||||
tsp.enrich(metadata)
|
||||
|
||||
assert len(metadata.media) == 1 # doesn't allow self-signed
|
||||
|
||||
# set self-signed on tsp
|
||||
tsp.allow_selfsigned = True
|
||||
tsp.enrich(metadata)
|
||||
|
||||
assert len(metadata.media) == 2
|
||||
|
||||
|
||||
def test_full_enriching(setup_module, sample_media, mocker, timestamp_response, filehash):
|
||||
mock_post = mocker.patch("requests.sessions.Session.post")
|
||||
mock_post.return_value.status_code = 200
|
||||
mock_decode_timestamp_response = mocker.patch(
|
||||
"auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response"
|
||||
)
|
||||
mock_decode_timestamp_response.return_value = timestamp_response
|
||||
|
||||
tsp: TimestampingEnricher = setup_module("timestamping_enricher", {"tsa_urls": ["http://timestamp.identrust.com"]})
|
||||
metadata = Metadata().set_url("https://example.com")
|
||||
sample_media.set("hash", filehash)
|
||||
metadata.add_media(sample_media)
|
||||
tsp.enrich(metadata)
|
||||
|
||||
assert metadata.get("timestamped") is True
|
||||
assert len(metadata.media) == 2 # the original 'sample_media' and the new 'timestamp_media'
|
||||
|
||||
timestamp_media = metadata.media[1]
|
||||
assert timestamp_media.filename == f"{tsp.tmp_dir}/hashes.txt"
|
||||
assert Path(timestamp_media.filename).read_text() == filehash
|
||||
|
||||
# we only have one authority file because we only used one TSA
|
||||
assert len(timestamp_media.get("timestamp_authority_files")) == 1
|
||||
timestamp_authority_file = timestamp_media.get("timestamp_authority_files")[0]
|
||||
assert Path(timestamp_authority_file.filename).read_bytes() == timestamp_response.time_stamp_token()
|
||||
|
||||
cert_chain = timestamp_authority_file.get("cert_chain")
|
||||
assert len(cert_chain) == 3
|
||||
assert cert_chain[0].filename == f"{tsp.tmp_dir}/1 – 85078758028491331763.crt"
|
||||
assert cert_chain[1].filename == f"{tsp.tmp_dir}/2 – 85078371663472981624.crt"
|
||||
assert cert_chain[2].filename == f"{tsp.tmp_dir}/3 – 13298821034946342390.crt"
|
||||
|
||||
|
||||
def test_full_enriching_multiple_tsa(setup_module, sample_media, mocker, timestamp_response, filehash):
|
||||
mock_post = mocker.patch("requests.sessions.Session.post")
|
||||
mock_post.return_value.status_code = 200
|
||||
|
||||
mock_decode_timestamp_response = mocker.patch(
|
||||
"auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response"
|
||||
)
|
||||
mock_decode_timestamp_response.return_value = timestamp_response
|
||||
|
||||
tsp: TimestampingEnricher = setup_module(
|
||||
"timestamping_enricher", {"tsa_urls": ["http://example.com/timestamp1", "http://example.com/timestamp2"]}
|
||||
)
|
||||
metadata = Metadata().set_url("https://example.com")
|
||||
sample_media.set("hash", filehash)
|
||||
metadata.add_media(sample_media)
|
||||
tsp.enrich(metadata)
|
||||
|
||||
assert metadata.get("timestamped") is True
|
||||
assert len(metadata.media) == 2 # the original 'sample_media' and the new 'timestamp_media'
|
||||
|
||||
timestamp_media = metadata.media[1]
|
||||
assert len(timestamp_media.get("timestamp_authority_files")) == 2
|
||||
for timestamp_token_media in timestamp_media.get("timestamp_authority_files"):
|
||||
assert Path(timestamp_token_media.filename).read_bytes() == timestamp_response.time_stamp_token()
|
||||
assert len(timestamp_token_media.get("cert_chain")) == 3
|
||||
|
||||
|
||||
def test_fails_for_digicert(setup_module, mocker, digicert_response):
|
||||
"""
|
||||
Digicert TSRs are not compliant with RFC 3161.
|
||||
See https://github.com/trailofbits/rfc3161-client/issues/104#issuecomment-2621960840
|
||||
"""
|
||||
mocker.patch("requests.sessions.Session.post", return_value=requests.Response())
|
||||
mocker.patch("requests.Response.raise_for_status")
|
||||
mocker.patch("requests.Response.content", new_callable=mocker.PropertyMock, return_value=digicert_response)
|
||||
|
||||
tsa_url = "http://timestamp.digicert.com"
|
||||
tsp: TimestampingEnricher = setup_module("timestamping_enricher")
|
||||
|
||||
data = b"4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef"
|
||||
with pytest.raises(ValueError) as e:
|
||||
tsp.sign_data(tsa_url, data)
|
||||
assert "ASN.1 parse error: ParseError" in str(e.value)
|
||||
|
||||
|
||||
@pytest.mark.download
|
||||
def test_download_tsr(setup_module):
|
||||
tsa_url = "http://timestamp.identrust.com"
|
||||
tsp: TimestampingEnricher = setup_module("timestamping_enricher")
|
||||
|
||||
data = b"4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef"
|
||||
result: TimeStampResponse = tsp.sign_data(tsa_url, data)
|
||||
assert isinstance(result, TimeStampResponse)
|
||||
|
||||
verified_root_cert = tsp.verify_signed(result, data)
|
||||
assert verified_root_cert.subject.rfc4514_string() == "CN=IdenTrust Commercial Root CA 1,O=IdenTrust,C=US"
|
||||
|
||||
# test downloading the cert
|
||||
cert_chain = tsp.save_certificate(result, verified_root_cert)
|
||||
assert len(cert_chain) == 3
|
||||
|
||||
|
||||
def test_verify_save(setup_module, timestamp_response):
|
||||
tsp: TimestampingEnricher = setup_module("timestamping_enricher")
|
||||
|
||||
verified_root_cert = tsp.verify_signed(
|
||||
timestamp_response, b"4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef"
|
||||
)
|
||||
assert verified_root_cert.subject.rfc4514_string() == "CN=IdenTrust Commercial Root CA 1,O=IdenTrust,C=US"
|
||||
|
||||
cert_chain = tsp.save_certificate(timestamp_response, verified_root_cert)
|
||||
assert len(cert_chain) == 3
|
||||
|
||||
assert cert_chain[0].filename == f"{tsp.tmp_dir}/1 – 85078758028491331763.crt"
|
||||
assert cert_chain[1].filename == f"{tsp.tmp_dir}/2 – 85078371663472981624.crt"
|
||||
assert cert_chain[2].filename == f"{tsp.tmp_dir}/3 – 13298821034946342390.crt"
|
||||
|
||||
|
||||
def test_order_crt_correctly(setup_module, wrong_order_timestamp_response):
|
||||
# reference: https://github.com/trailofbits/rfc3161-client/issues/104#issuecomment-2711244010
|
||||
tsp: TimestampingEnricher = setup_module("timestamping_enricher")
|
||||
|
||||
# get the certificates, make sure the reordering is working:
|
||||
|
||||
ordered_certs = tsp.tst_certs(wrong_order_timestamp_response)
|
||||
assert len(ordered_certs) == 2
|
||||
assert ordered_certs[0].subject.rfc4514_string() == "CN=TrustID Timestamp Authority,O=IdenTrust,C=US"
|
||||
assert ordered_certs[1].subject.rfc4514_string() == "CN=TrustID Timestamping CA 3,O=IdenTrust,C=US"
|
||||
|
||||
|
||||
def test_invalid_tsa_invalid_response(setup_module, mocker):
|
||||
mocker.patch("requests.sessions.Session.post", return_value=requests.Response())
|
||||
raise_for_status = mocker.patch("requests.Response.raise_for_status")
|
||||
raise_for_status.side_effect = requests.exceptions.HTTPError("404 Client Error")
|
||||
tsp = setup_module("timestamping_enricher")
|
||||
|
||||
with pytest.raises(requests.exceptions.HTTPError, match="404 Client Error"):
|
||||
tsp.sign_data("http://bellingcat.com/page-not-found/", b"my-message")
|
||||
|
||||
|
||||
def test_fail_on_selfsigned_cert(setup_module, selfsigned_response):
|
||||
tsp = setup_module("timestamping_enricher")
|
||||
root_cert = tsp.verify_signed(selfsigned_response, b"my-message")
|
||||
assert root_cert is None
|
||||
Ładowanie…
Reference in New Issue