From 4c1c8953ca77c843a72508a4c4c0eef994491ca8 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Wed, 29 Jan 2025 12:20:52 +0100 Subject: [PATCH 01/20] Add unit tests for timestamping_enricher --- .../timestamping_enricher.py | 25 +++++++---- tests/data/timestamp_token_digicert_com.crt | Bin 0 -> 5964 bytes tests/enrichers/test_timestamping_enricher.py | 41 ++++++++++++++++++ 3 files changed, 58 insertions(+), 8 deletions(-) create mode 100644 tests/data/timestamp_token_digicert_com.crt create mode 100644 tests/enrichers/test_timestamping_enricher.py diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index a7a0aee..4d5dd20 100644 --- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -6,11 +6,11 @@ from importlib.metadata import version from asn1crypto.cms import ContentInfo from certvalidator import CertificateValidator, ValidationContext from asn1crypto import pem +from asn1crypto.core import Asn1Value import certifi from auto_archiver.core import Enricher from auto_archiver.core import Metadata, ArchivingContext, Media -from auto_archiver.core import Extractor class TimestampingEnricher(Enricher): @@ -45,13 +45,10 @@ class TimestampingEnricher(Enricher): from slugify import slugify for tsa_url in self.tsa_urls: try: - signing_settings = SigningSettings(tsp_server=tsa_url, digest_algorithm=DigestAlgorithm.SHA256) - signer = TSPSigner() message = bytes(data_to_sign, encoding='utf8') - # send TSQ and get TSR from the TSA server - signed = signer.sign(message=message, signing_settings=signing_settings) + signed = self.sign_data(tsa_url, message) # fail if there's any issue with the certificates, uses certifi list of trusted CAs - TSPVerifier(certifi.where()).verify(signed, message=message) + self.verify_signed(signed, message) # download and verify timestamping certificate cert_chain = self.download_and_verify_certificate(signed) # continue with saving the timestamp token @@ -72,9 +69,22 @@ class TimestampingEnricher(Enricher): else: logger.warning(f"No successful timestamps for {url=}") + def verify_signed(self, signed: bytes, message: bytes) -> None: + verifier = TSPVerifier(certifi.where()) + verifier.verify(signed, message=message) + + def sign_data(self, tsa_url: str, bytes_data: bytes) -> bytes: + signing_settings = SigningSettings(tsp_server=tsa_url, digest_algorithm=DigestAlgorithm.SHA256) + signer = TSPSigner() + # send TSQ and get TSR from the TSA server + return signer.sign(message=bytes_data, signing_settings=signing_settings) + + def load_tst_certs(self, signed: bytes) -> list[Asn1Value]: + return ContentInfo.load(signed)["content"]["certificates"] + def download_and_verify_certificate(self, signed: bytes) -> list[Media]: # returns the leaf certificate URL, fails if not set - tst = ContentInfo.load(signed) + certificates = self.load_tst_certs(signed) trust_roots = [] with open(certifi.where(), 'rb') as f: @@ -82,7 +92,6 @@ class TimestampingEnricher(Enricher): trust_roots.append(der_bytes) context = ValidationContext(trust_roots=trust_roots) - certificates = tst["content"]["certificates"] first_cert = certificates[0].dump() intermediate_certs = [] for i in range(1, len(certificates)): # cannot use list comprehension [1:] diff --git a/tests/data/timestamp_token_digicert_com.crt b/tests/data/timestamp_token_digicert_com.crt new file mode 100644 index 0000000000000000000000000000000000000000..592edf4ae0e96057379dd362f8671e6385a8a8d4 GIT binary patch literal 5964 zcmchbc|6qX+sDmfmwn4Rb{flf#!l9;hGgGDvea0z%tEqGvZPX3LSZHfkx)@~4$2nU zMI>9Ihzg0GQ8`EFcb?~*-|Kn3&ipmkeP8!=U-!)Y`M$r`1rWH5sAvzw8pW>d15tyC z1TIy8z@-QVK@d#9J}TN{u|^5==&y^olRim-$?-=fQ9#-jr?yx2w)}(@;Dus z00vRid1G=urwSFN0aZ?Zvktir-)vnGI^U|hgZe_oyb(ptsNUAfn`p}g* z3VUGR9v0J4)jMd>Bk5>YcQH0(Z4tq37HJVO7#4Fj{Y?DK(wFt2o_j3kjEW_TD8{e`N#EwGiS>A+keWFD3 z;JQ=t^(L2%axUv|;v(RX4|)Up;Dz?dz7HLQ3##gbL|?KZ5Ai6v3U$CB3E_c{4<15e zaKj-in=K`&W?2g=7?*^E;SCp&0ess=j-F&Kml;>SYkUC13*Vb1E&H}Qc^=` z0NBnr47wlS0%)kHPDC?+$nNxel`pvCAp+$FIHTCBI7j@<&dFh<@R@SB3*NA#?5F+! z|BeGAloQIHVW|ZW(>~+iC<-Pe;K|Ik3aNL;|gjxX>fH~2OXcB9D5{nH`laq4~ z@|BnMAZyZ{tVvmSe?PfhA^en~p~6oQ{?OoF38Hjl$;*N0%ezMbfuPZ6f#@hebaw;* z#d81L$=@9l@XJns{qJ7jSkv8n+;$Sd{NniEri9%qSn|V&ELaeQ#$5qHe1_AMN(`&v z2=yU!4udugcB%bJT(O+!fEpp>v|2Sm%bFeh$+9}(fb|@$?S&D; z6bt{a-N8#q+-7gNtkzn5hHb@YWjp=qsOzr->a$U<94Xgvh16p2qOB3}`>kv)ybUWI zJ<~T2B{oKw7oi`t>`Q%PX;|I4E)!foQR^{7*Y(jF6jT1xIK856KCS25e$THV&pxJg z+029LJzj)8zUX|qs$m^St; zZlw625RDsTv)R`H8tzjPn(np*mkk?1ac`*=uv^#gD5%!8=(32>$W|BUI}| zhJoyA0P5irYW|>`z8zX^fs^|%9>f*?_7EGeNi~}Da8t2b9+_VyWPYXnd@$O_P4RL1ukHG+&SY@UcgnpdtU z7hba<4fdXWo^5*5k67AycEj;A3rUYd*EKJG_2ifE)f)-)ky(P-O!9(9*yzPEX@2W= z1>o>4)PtB8EFUms_us>#V$KV5Pb|F9Xy%G6+7H*%9XoO9t%b(&pl$!0q|8EJ*+bl% zAeHCE6sf+EMfAkTQCKpM6qgJ+Z`i%iVYSb_;)((!Qeuv%Ce39$4-ChM<xII=|vIv8EHgSl==cCdh zwc36s4WoN9r^4WVmRjxrFh&4cZ_W86lO>Q`=Yt|HYQQ|_;BPilT{!j@|Z(Bc=1oDFcVkLB9a zf+Q&zW31mln|FTd05dgh3HrgTq3)lcn6U%w}xefYq~l?#(AhoL`$rZ3dj*cSj2S72Vj)Pd%@B z?CCK5%u%Nyzql_kFfj$hiLMIPH2>x!rpNgiXb1(IwS~6b9GHtBVXINz(aSz$c^bYV*cH8}Gs$F)M{ za_9MPlGX{!1Lp(ro-aC=gK|Z_G)NHiN-`vPi$n#)8!N{TEPT!`JDm}oYa3fcr{6Zw ze33YpzPWijE9hSHlg38xz>!wADB9+VoUXVy(Q5Nto=V)^9t?jLPFX-m29G2WBeg%)7f`t6audfiTg~e zV2W;o%xksOL++a1-E2-h^N!9{UNuf9BvfVo#^g(pEq=+qCmU}*q`Pjyha*-DvT)J@ z64K(M{v2ln_9rSmO75Rs;mUAm4_H>NPp6oGkzf5bkDcHoc~!l%X;71jElnrF-=*qQ zsts_YBOV;5WumK%9%0+A&wIXVH&O%rLKsI*KogcJx}2LfdPE{A=%?{u@sVXuyzXtO zI9fyaY$Q+KIi|1aX`gMZ;et_L8dezLGlQFja(Wzx=GYkdN0n80Nbt|^2E)Dr1j>tK zV&Q%gi}5=5rHvk$xuww~uW7Wo9rqH8fn0HU{(p%j3JCuo7Vq!G(!pSGLF7`*SP%9c zT0d(pB%ngBMSrik5C{ah2>V%c{Ttx?J7(3-UdEgpuby^3bPx~*(DI8-`C_tdLK-XQ zhaH+GrSdiGi}0CUC1-eJ_B8{7tGszCoC~(C*Zla4%1^Jfv-D284pJVpWU>@&&E~ij zQI}I_$LU}bbnj$xKvQ`YF2fsxOnJ_%$1FQRSf!4c#j5)SzLsI&eGeOGgYmmd$vj(h zIs7zvUC8aa!UAQh%c2uc%|csuSFwV&Ygwq*`{1WW1zvV`Io9@<7~qxW`M`+n`}Q-= zQ+M0UU1k*L5FIa)UB)gYMZKuFlCFLE^lu?gD$1H4dULBlm@f|ZE9c2lhR1&*Xfj_3 zD6%2wAcsbwm;FQOrY>hFB zX??MTicG!z@Q8+#2fc273-Ti^6DClK&Tzie-;&mZ%F9_Qt0PwiE(A#H9J^EI&aIcq zgFhbeIjVH?jmK^I=Lyu4wUx7|Jnl!ggF%x_DGWmV<Q=FFjqL6Dd%)-aNtSC0oylGJ1H4FBYB+$~3 zbh9jiPCVn&G+Hj5U*^+DVLAT{?=G{{$jnmuUiAN979{{7Giwj9{*p&~fu+B2^WMj+ z9R!4oMPh!wP}Zv7Q>Om_*6Cf!klEx1`20gQp>|i1*`&P7l>e*x`6)(Yy&Wljuf4E< z{tr570%SfB$$W}c*p2kR(&<0L_yR0DVPH@?fJ}8NDETYJpPnud0tiGAK&(d{wV6|K zCFWvzH7YoV*6Qu3Ycq>Uang(DTnc%rh1cVF(AL85^5TR2YI)c{vK8z0w-fzZsva+m(@=P14FD4~V4Ad=C8jE*exd@2ZA zu!LU!T;v;vg5`QS`rGp>3I`Y0AWz~7dQG+WD;jvCtxF$~Y%eY54QRY68{6U={uu0d z)cS(|66uJ>S3K>>m>mE1QP@SrV|-vp2duQ%p2(=}-jXidSnlx2#;04N$QS?2Uqt3( zM4-2bkMK9vN7gbD5yNUD%N22HTTai97IXh*mV>DT73SF8aglueI0Zp~-~j?8h}@L} zg8!8c*I)P9_H+)3(a!;%-#c%#5X8P;8i@ekA9IulCUU!MZ{rVu?{Nk*Aou_tfZSn# zvXAzidw+biSM$vD=Rl()aiRmUPWph{AEBTKVJK`W|4W*MqWr5OfR0&d{dGG<>OF6@ zG?JcbLDlH64+j^Zyf?ZWLJ|-M$cTZ$@c5j96?Zv76up`VP%>zX+aAJ&0^EuXn(E}+b8DD@i0ZWDPUCdBs(J~MQhd+Ax@f+ zeoj1KD^F8~$2q--khCbX zMrY_MN{P)ZmpjvcIn>GqIv#Eit}D>%b`x#i`*HXi&xf0twq@tX;5gKJrvD=YiX)&| ziMq;;49dGUxD3bIPwmJqo`c7-n^qUEN|tflsgr2$T}s(*a<#-fD2eLa$|l_{UuWn= z80DtHksM2{OWWMudbNyGc4J@_gkzn=M|37!xzdEXRfFAi;J0zJypG(%;WUizHIrS@ zQz;^lX!Xr)i=ztm8~FW??D!gC^4$xcuBmy-Za;Ee*u3s!D6?LUPK_#ovt3I`1&Pr#=VevwBV|WgT zzZ2(huQlE`;H1X22~W Date: Tue, 11 Feb 2025 15:26:40 +0000 Subject: [PATCH 02/20] Fix timestamping enricher for new module structure (temp paths) --- tests/enrichers/test_timestamping_enricher.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/enrichers/test_timestamping_enricher.py b/tests/enrichers/test_timestamping_enricher.py index 3c978f0..b8351d3 100644 --- a/tests/enrichers/test_timestamping_enricher.py +++ b/tests/enrichers/test_timestamping_enricher.py @@ -22,14 +22,12 @@ def test_sign_data(setup_module): def test_tsp_enricher_download_syndication(setup_module, digicert): tsp: TimestampingEnricher = setup_module("timestamping_enricher") - try: - cert_chain = tsp.download_and_verify_certificate(digicert) - assert len(cert_chain) == 3 - assert cert_chain[0].filename == "/var/folders/h7/g67pz_kx67q7qxzzrrhvry5r0000gn/T/74515005589773707779.crt" - assert cert_chain[1].filename == "/var/folders/h7/g67pz_kx67q7qxzzrrhvry5r0000gn/T/95861100433808324400.crt" - assert cert_chain[2].filename == "/var/folders/h7/g67pz_kx67q7qxzzrrhvry5r0000gn/T/15527051335772373346.crt" - except Exception as e: - pytest.fail(f"Verification failed: {e}") + + cert_chain = tsp.download_and_verify_certificate(digicert) + assert len(cert_chain) == 3 + assert cert_chain[0].filename == f"{tsp.tmp_dir}/74515005589773707779.crt" + assert cert_chain[1].filename == f"{tsp.tmp_dir}/95861100433808324400.crt" + assert cert_chain[2].filename == f"{tsp.tmp_dir}/15527051335772373346.crt" def test_tst_cert_valid(setup_module, digicert): From d0c379a3baa8495f89a3bbb0997c2babde3a90e9 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Tue, 11 Feb 2025 18:18:19 +0000 Subject: [PATCH 03/20] WIP - timestamping enricher --- .../timestamping_enricher.py | 121 ++++++++++++++---- tests/enrichers/test_timestamping_enricher.py | 12 +- 2 files changed, 107 insertions(+), 26 deletions(-) diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index 1a0f932..90860dc 100644 --- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -1,16 +1,21 @@ import os from loguru import logger -from tsp_client import TSPSigner, SigningSettings, TSPVerifier -from tsp_client.algorithms import DigestAlgorithm -from importlib.metadata import version -from asn1crypto.cms import ContentInfo -from certvalidator import CertificateValidator, ValidationContext -from asn1crypto import pem -from asn1crypto.core import Asn1Value -import certifi +from importlib.metadata import version + +import requests +from rfc3161_client import ( + TimestampRequestBuilder, + TimeStampResponse, + decode_timestamp_response, + VerifierBuilder +) +from rfc3161_client import VerificationError as Rfc3161VerificationError +from rfc3161_client.base import HashAlgorithm +import certifi from auto_archiver.core import Enricher from auto_archiver.core import Metadata, Media +from auto_archiver.version import __version__ class TimestampingEnricher(Enricher): """ @@ -21,6 +26,22 @@ class TimestampingEnricher(Enricher): See https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710 for list of timestamp authorities. """ + def setup(self): + self.session = requests.Session() + self.session.headers.update( + { + "Content-Type": "application/timestamp-query", + "User-Agent": f"Auto-Archiver {__version__}", + "Accept": "application/timestamp-reply", + } + ) + + def __del__(self) -> None: + """ + Terminates the underlying network session. + """ + self.session.close() + def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() logger.debug(f"RFC3161 timestamping existing files for {url=}") @@ -32,8 +53,7 @@ class TimestampingEnricher(Enricher): logger.warning(f"No hashes found in {url=}") return - tmp_dir = self.tmp_dir - hashes_fn = os.path.join(tmp_dir, "hashes.txt") + hashes_fn = os.path.join(self.tmp_dir, "hashes.txt") data_to_sign = "\n".join(hashes) with open(hashes_fn, "w") as f: @@ -45,14 +65,15 @@ class TimestampingEnricher(Enricher): for tsa_url in self.tsa_urls: try: message = bytes(data_to_sign, encoding='utf8') - signed = self.sign_data(tsa_url, message) + signed: TimeStampResponse = self.sign_data(tsa_url, message) # fail if there's any issue with the certificates, uses certifi list of trusted CAs self.verify_signed(signed, message) # download and verify timestamping certificate cert_chain = self.download_and_verify_certificate(signed) # continue with saving the timestamp token - tst_fn = os.path.join(tmp_dir, f"timestamp_token_{slugify(tsa_url)}") - with open(tst_fn, "wb") as f: f.write(signed) + tst_fn = os.path.join(self.tmp_dir, f"timestamp_token_{slugify(tsa_url)}") + with open(tst_fn, "wb") as f: + f.write(signed) timestamp_tokens.append(Media(filename=tst_fn).set("tsa", tsa_url).set("cert_chain", cert_chain)) except Exception as e: logger.warning(f"Error while timestamping {url=} with {tsa_url=}: {e}") @@ -68,17 +89,73 @@ class TimestampingEnricher(Enricher): else: logger.warning(f"No successful timestamps for {url=}") - def verify_signed(self, signed: bytes, message: bytes) -> None: - verifier = TSPVerifier(certifi.where()) - verifier.verify(signed, message=message) + def verify_signed(self, timestamp_response: TimeStampResponse, signature: bytes) -> None: + """ + Verify a Signed Timestamp using the TSA provided by the Trusted Root. + """ + cert_authorities = self._trusted_root.get_timestamp_authorities() + valid = False + for certificate_authority in cert_authorities: + certificates = certificate_authority.certificates(allow_expired=True) - def sign_data(self, tsa_url: str, bytes_data: bytes) -> bytes: - signing_settings = SigningSettings(tsp_server=tsa_url, digest_algorithm=DigestAlgorithm.SHA256) - signer = TSPSigner() - # send TSQ and get TSR from the TSA server - return signer.sign(message=bytes_data, signing_settings=signing_settings) + builder = VerifierBuilder() + for certificate in certificates: + builder.add_root_certificate(certificate) + + verifier = builder.build() + try: + verifier.verify(timestamp_response, signature) + except Rfc3161VerificationError as e: + logger.debug("Unable to verify Timestamp with CA.") + logger.exception(e) + continue + + if ( + certificate_authority.validity_period_start + and certificate_authority.validity_period_end + ): + if ( + certificate_authority.validity_period_start + <= timestamp_response.tst_info.gen_time + < certificate_authority.validity_period_end + ): + return TimestampVerificationResult( + source=TimestampSource.TIMESTAMP_AUTHORITY, + time=timestamp_response.tst_info.gen_time, + ) + + logger.debug( + "Unable to verify Timestamp because not in CA time range." + ) + else: + logger.debug( + "Unable to verify Timestamp because no validity provided." + ) + + return None + + def sign_data(self, tsa_url: str, bytes_data: bytes) -> TimeStampResponse: + # see https://github.com/sigstore/sigstore-python/blob/99948d5b80525a5a104e904ffea58169dc6e0629/sigstore/_internal/timestamp.py#L84-L121 + + timestamp_request = ( + TimestampRequestBuilder().data(bytes_data).nonce(nonce=True).build() + ) + try: + response = self.session.post(tsa_url, data=timestamp_request.as_bytes(), timeout=10) + response.raise_for_status() + except requests.RequestException as e: + logger.error(f"Error while sending request to {tsa_url=}: {e}") + raise + + # Check that we can parse the response but do not *verify* it + try: + timestamp_response = decode_timestamp_response(response.content) + except ValueError as e: + logger.error(f"Invalid timestamp response from server {tsa_url}: {e}") + raise + return timestamp_response - def load_tst_certs(self, signed: bytes) -> list[Asn1Value]: + def load_tst_certs(self, signed: bytes): return ContentInfo.load(signed)["content"]["certificates"] def download_and_verify_certificate(self, signed: bytes) -> list[Media]: diff --git a/tests/enrichers/test_timestamping_enricher.py b/tests/enrichers/test_timestamping_enricher.py index b8351d3..a90d4bd 100644 --- a/tests/enrichers/test_timestamping_enricher.py +++ b/tests/enrichers/test_timestamping_enricher.py @@ -1,6 +1,10 @@ import pytest from auto_archiver.modules.timestamping_enricher.timestamping_enricher import TimestampingEnricher - +from rfc3161_client import ( + TimestampRequestBuilder, + TimeStampResponse, + decode_timestamp_response, +) @pytest.fixture def digicert(): @@ -9,11 +13,11 @@ def digicert(): @pytest.mark.download def test_sign_data(setup_module): - tsa_url = "http://timestamp.digicert.com" + tsa_url = "http://timestamp.identrust.com" tsp: TimestampingEnricher = setup_module("timestamping_enricher") data = b"4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef" - result: bytes = tsp.sign_data(tsa_url, data) - assert isinstance(result, bytes) + result: TimeStampResponse = tsp.sign_data(tsa_url, data) + assert isinstance(result, TimeStampResponse) try: tsp.verify_signed(result, data) From 6987a4827e911ad3ea5f35d537c35126123a724e Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Tue, 25 Feb 2025 11:57:20 +0000 Subject: [PATCH 04/20] Set poetry packages - remove tsp_client and update cryptography --- poetry.lock | 115 ++++++++++++++++++++++++++++--------------------- pyproject.toml | 3 +- 2 files changed, 67 insertions(+), 51 deletions(-) diff --git a/poetry.lock b/poetry.lock index 83b2860..31adc90 100644 --- a/poetry.lock +++ b/poetry.lock @@ -408,6 +408,7 @@ description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" groups = ["main"] +markers = "os_name == \"nt\" and implementation_name != \"pypy\" or platform_python_implementation != \"PyPy\"" files = [ {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, @@ -613,48 +614,52 @@ markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\"", [[package]] name = "cryptography" -version = "41.0.7" +version = "43.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"}, - {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"}, - {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"}, - {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"}, - {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"}, - {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"}, - {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"}, - {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"}, - {file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"}, - {file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"}, - {file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"}, - {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, - {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, - {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, - {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, - {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, - {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, - {file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"}, - {file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"}, + {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e1ce50266f4f70bf41a2c6dc4358afadae90e2a1e5342d3c08883df1675374f"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:443c4a81bb10daed9a8f334365fe52542771f25aedaf889fd323a853ce7377d6"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:74f57f24754fe349223792466a709f8e0c093205ff0dca557af51072ff47ab18"}, + {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9762ea51a8fc2a88b70cf2995e5675b38d93bf36bd67d91721c309df184f49bd"}, + {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:81ef806b1fef6b06dcebad789f988d3b37ccaee225695cf3e07648eee0fc6b73"}, + {file = "cryptography-43.0.3-cp37-abi3-win32.whl", hash = "sha256:cbeb489927bd7af4aa98d4b261af9a5bc025bd87f0e3547e11584be9e9427be2"}, + {file = "cryptography-43.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:f46304d6f0c6ab8e52770addfa2fc41e6629495548862279641972b6215451cd"}, + {file = "cryptography-43.0.3-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:8ac43ae87929a5982f5948ceda07001ee5e83227fd69cf55b109144938d96984"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405"}, + {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16"}, + {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73"}, + {file = "cryptography-43.0.3-cp39-abi3-win32.whl", hash = "sha256:d56e96520b1020449bbace2b78b603442e7e378a9b3bd68de65c782db1507995"}, + {file = "cryptography-43.0.3-cp39-abi3-win_amd64.whl", hash = "sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d03b5621a135bffecad2c73e9f4deb1a0f977b9a8ffe6f8e002bf6c9d07b918c"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a2a431ee15799d6db9fe80c82b055bae5a752bef645bba795e8e52687c69efe3"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:281c945d0e28c92ca5e5930664c1cefd85efe80e5c0d2bc58dd63383fda29f83"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f18c716be16bc1fea8e95def49edf46b82fccaa88587a45f8dc0ff6ab5d8e0a7"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4a02ded6cd4f0a5562a8887df8b3bd14e822a90f97ac5e544c162899bc467664"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:53a583b6637ab4c4e3591a15bc9db855b8d9dee9a669b550f311480acab6eb08"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1ec0bcf7e17c0c5669d881b1cd38c4972fade441b27bda1051665faaa89bdcaa"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff"}, + {file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"}, ] [package.dependencies] -cffi = ">=1.12" +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} [package.extras] docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] -docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] nox = ["nox"] -pep8test = ["black", "check-sdist", "mypy", "ruff"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] sdist = ["build"] ssh = ["bcrypt (>=3.1.5)"] -test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test = ["certifi", "cryptography-vectors (==43.0.3)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] [[package]] @@ -1625,6 +1630,7 @@ description = "C parser in Python" optional = false python-versions = ">=3.8" groups = ["main"] +markers = "os_name == \"nt\" and implementation_name != \"pypy\" or platform_python_implementation != \"PyPy\"" files = [ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, @@ -2119,6 +2125,37 @@ files = [ [package.dependencies] six = ">=1.7.0" +[[package]] +name = "rfc3161-client" +version = "1.0.0" +description = "" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "rfc3161_client-1.0.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:702d83cd6f4bcaf648ea20c5bc64e69ca8ecc99e49dd6ecc82b574739d4792f1"}, + {file = "rfc3161_client-1.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:d4474600be7aa12d37838bff6d218326b9b08059ee4afa757ce16151f168b76a"}, + {file = "rfc3161_client-1.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a3374d72945bae12ceac2a68e39953ed95ed2a35121cd5f3d26f90851f53c26"}, + {file = "rfc3161_client-1.0.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:93c56d7de2f43be6aa4a1d1c8723982453517b77512e9b6dd46cc2343c92b3a5"}, + {file = "rfc3161_client-1.0.0-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:311ae7278cf914b448ebf707be566161025c0b26180629c78fb0efb5863edd3a"}, + {file = "rfc3161_client-1.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f46be2701f60d6649e0ff4bb094e0e0056f934324f7e3e1d2ac811964a171c1b"}, + {file = "rfc3161_client-1.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f88e19cb92fcf9eb67494d006b4bd3c7a206cb1c6b207b6bd3b56bea767caa7f"}, + {file = "rfc3161_client-1.0.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:f0fda030a58ba1840aecf56d234458cd7ace7437d4a03d650dcbbfbe328528fc"}, + {file = "rfc3161_client-1.0.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:c593b0a990ee48f3fb2f92dd5a6c41b81930c0a7749161aff5c6f173a3ce75ec"}, + {file = "rfc3161_client-1.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:67dd79572d7cf6c658acd0d0bde2892bf0a5b6ec22f68156f23ef117bee27bc8"}, + {file = "rfc3161_client-1.0.0-cp39-abi3-win32.whl", hash = "sha256:90e5969b82e0939821c06b2adb4322c6a55337337a169df7ecfcb808c7c2a14e"}, + {file = "rfc3161_client-1.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:1dbc6585957dadcfec531779c48f15e51242c91e94082b99f56a8426cacd1974"}, + {file = "rfc3161_client-1.0.0.tar.gz", hash = "sha256:626619d12154c793c034fc72fdb05043b69cb4ca278cdb0b889194e87af7172b"}, +] + +[package.dependencies] +cryptography = ">=43,<45" + +[package.extras] +dev = ["maturin (>=1.7,<2.0)", "rfc3161-client[doc,lint,test]"] +lint = ["interrogate", "ruff (>=0.7,<0.9)"] +test = ["coverage[toml]", "pretend", "pytest", "pytest-cov"] + [[package]] name = "rich" version = "13.9.4" @@ -2735,26 +2772,6 @@ outcome = ">=1.2.0" trio = ">=0.11" wsproto = ">=0.14" -[[package]] -name = "tsp-client" -version = "0.2.0" -description = "An IETF Time-Stamp Protocol (TSP) (RFC 3161) client" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "tsp-client-0.2.0.tar.gz", hash = "sha256:6e66148dd116322eb44a7484e5ad33bbe640b997343c443de9cc70fc5eb19987"}, - {file = "tsp_client-0.2.0-py3-none-any.whl", hash = "sha256:0b790d10a68d66782c13f1d7cc7f5206df26b49826c1da80944b7c05b1731784"}, -] - -[package.dependencies] -asn1crypto = ">=0.24.0" -pyOpenSSL = ">=20.0.0" -requests = ">=2.18.4" - -[package.extras] -tests = ["build", "coverage", "mypy", "ruff", "wheel"] - [[package]] name = "typing-extensions" version = "4.12.2" @@ -3184,4 +3201,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "2d0a953383901fe12e97f6f56a76a9d8008788695425792eedbf739a18585188" +content-hash = "1a548dd3eeeaf1fb408b8339bf84ba31c1362a6f4c3c0aa98cdcb82ece256acc" diff --git a/pyproject.toml b/pyproject.toml index 3c64eae..c811563 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,6 @@ dependencies = [ "tqdm (>=0.0.0)", "jinja2 (>=0.0.0)", "pyOpenSSL (==24.2.1)", - "cryptography (>=41.0.0,<42.0.0)", "boto3 (>=1.28.0,<2.0.0)", "dataclasses-json (>=0.0.0)", "yt-dlp (>=2025.1.26,<2026.0.0)", @@ -53,10 +52,10 @@ dependencies = [ "jsonlines (>=0.0.0)", "pysubs2 (>=0.0.0)", "retrying (>=0.0.0)", - "tsp-client (>=0.0.0)", "certvalidator (>=0.0.0)", "rich-argparse (>=1.6.0,<2.0.0)", "ruamel-yaml (>=0.18.10,<0.19.0)", + "rfc3161-client (>=1.0.0,<2.0.0)", ] [tool.poetry.group.dev.dependencies] From 898faf6fe457ba4d2ae28f007617b571c05744be Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Tue, 25 Feb 2025 12:08:08 +0000 Subject: [PATCH 05/20] Further WIP - currently working on verify_signed --- .../timestamping_enricher/__manifest__.py | 5 ++ .../timestamping_enricher.py | 75 +++++++------------ tests/enrichers/test_timestamping_enricher.py | 8 +- 3 files changed, 38 insertions(+), 50 deletions(-) diff --git a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py index 6ad9c57..e945350 100644 --- a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py +++ b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py @@ -36,6 +36,11 @@ "http://tss.accv.es:8318/tsa", ], "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.", + }, + "cert_authorities": { + "default": None, + "help": "Path to a file containing trusted Certificate Authorities (CAs) in PEM format. If empty, the default system authorities are used.", + "type": "str", } }, "description": """ diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index 90860dc..0031210 100644 --- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -12,6 +12,8 @@ from rfc3161_client import ( ) from rfc3161_client import VerificationError as Rfc3161VerificationError from rfc3161_client.base import HashAlgorithm +from rfc3161_client.tsp import SignedData +from cryptography import x509 import certifi from auto_archiver.core import Enricher from auto_archiver.core import Metadata, Media @@ -69,7 +71,7 @@ class TimestampingEnricher(Enricher): # fail if there's any issue with the certificates, uses certifi list of trusted CAs self.verify_signed(signed, message) # download and verify timestamping certificate - cert_chain = self.download_and_verify_certificate(signed) + cert_chain = self.download_certificate(signed) # continue with saving the timestamp token tst_fn = os.path.join(self.tmp_dir, f"timestamp_token_{slugify(tsa_url)}") with open(tst_fn, "wb") as f: @@ -93,46 +95,31 @@ class TimestampingEnricher(Enricher): """ Verify a Signed Timestamp using the TSA provided by the Trusted Root. """ - cert_authorities = self._trusted_root.get_timestamp_authorities() - valid = False - for certificate_authority in cert_authorities: - certificates = certificate_authority.certificates(allow_expired=True) + trusted_root_path = self.cert_authorities or certifi.where() + cert_authorities = [] + + with open(trusted_root_path, 'rb') as f: + cert_authorities = x509.load_pem_x509_certificates(f.read()) + + if not cert_authorities: + raise ValueError(f"No trusted roots found in {trusted_root_path}.") + + + valid = False + for certificate in cert_authorities: builder = VerifierBuilder() - for certificate in certificates: - builder.add_root_certificate(certificate) + builder.add_root_certificate(certificate) verifier = builder.build() try: verifier.verify(timestamp_response, signature) + return certificate except Rfc3161VerificationError as e: - logger.debug("Unable to verify Timestamp with CA.") - logger.exception(e) + logger.debug(f"Unable to verify Timestamp with CA {certificate.subject}: {e}") continue - - if ( - certificate_authority.validity_period_start - and certificate_authority.validity_period_end - ): - if ( - certificate_authority.validity_period_start - <= timestamp_response.tst_info.gen_time - < certificate_authority.validity_period_end - ): - return TimestampVerificationResult( - source=TimestampSource.TIMESTAMP_AUTHORITY, - time=timestamp_response.tst_info.gen_time, - ) - - logger.debug( - "Unable to verify Timestamp because not in CA time range." - ) - else: - logger.debug( - "Unable to verify Timestamp because no validity provided." - ) - - return None + + return False def sign_data(self, tsa_url: str, bytes_data: bytes) -> TimeStampResponse: # see https://github.com/sigstore/sigstore-python/blob/99948d5b80525a5a104e904ffea58169dc6e0629/sigstore/_internal/timestamp.py#L84-L121 @@ -155,26 +142,16 @@ class TimestampingEnricher(Enricher): raise return timestamp_response - def load_tst_certs(self, signed: bytes): - return ContentInfo.load(signed)["content"]["certificates"] + def load_tst_certs(self, tsp_response: TimeStampResponse): + signed_data: SignedData = tsp_response.signed_data + certs = signed_data.certificates + - def download_and_verify_certificate(self, signed: bytes) -> list[Media]: + def download_certificate(self, tsp_response: TimeStampResponse) -> list[Media]: # returns the leaf certificate URL, fails if not set - certificates = self.load_tst_certs(signed) - trust_roots = [] - with open(certifi.where(), 'rb') as f: - for _, _, der_bytes in pem.unarmor(f.read(), multiple=True): - trust_roots.append(der_bytes) - context = ValidationContext(trust_roots=trust_roots) + certificates = self.load_tst_certs(tsp_response) - first_cert = certificates[0].dump() - intermediate_certs = [] - for i in range(1, len(certificates)): # cannot use list comprehension [1:] - intermediate_certs.append(certificates[i].dump()) - - validator = CertificateValidator(first_cert, intermediate_certs=intermediate_certs, validation_context=context) - path = validator.validate_usage({'digital_signature'}, extended_key_usage={'time_stamping'}) cert_chain = [] for cert in path: diff --git a/tests/enrichers/test_timestamping_enricher.py b/tests/enrichers/test_timestamping_enricher.py index a90d4bd..7d30e2f 100644 --- a/tests/enrichers/test_timestamping_enricher.py +++ b/tests/enrichers/test_timestamping_enricher.py @@ -15,15 +15,21 @@ def digicert(): def test_sign_data(setup_module): tsa_url = "http://timestamp.identrust.com" tsp: TimestampingEnricher = setup_module("timestamping_enricher") + data = b"4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef" result: TimeStampResponse = tsp.sign_data(tsa_url, data) assert isinstance(result, TimeStampResponse) + try: - tsp.verify_signed(result, data) + valid_root = tsp.verify_signed(result, data) + assert valid_root.subject == "CN=Entrust Root Certification Authority - G2, OU=(c) 2009 Entrust, Inc. - for authorized use only, OU=See www.entrust.net/legal-terms, O=Entrust, Inc., C=" except Exception as e: pytest.fail(f"Verification failed: {e}") + # test downloading the cert + cert_chain = tsp.download_and_verify_certificate(result) + def test_tsp_enricher_download_syndication(setup_module, digicert): tsp: TimestampingEnricher = setup_module("timestamping_enricher") From afc117a229734a6dfe5c556a7db03e81e4f3be3f Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Wed, 26 Feb 2025 09:33:56 +0000 Subject: [PATCH 06/20] Get downloading certs working --- .../timestamping_enricher/timestamping_enricher.py | 11 ++++++----- tests/enrichers/test_timestamping_enricher.py | 6 ++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index 0031210..c138cee 100644 --- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -14,6 +14,7 @@ from rfc3161_client import VerificationError as Rfc3161VerificationError from rfc3161_client.base import HashAlgorithm from rfc3161_client.tsp import SignedData from cryptography import x509 +from cryptography.hazmat.primitives import serialization import certifi from auto_archiver.core import Enricher from auto_archiver.core import Metadata, Media @@ -106,7 +107,6 @@ class TimestampingEnricher(Enricher): raise ValueError(f"No trusted roots found in {trusted_root_path}.") - valid = False for certificate in cert_authorities: builder = VerifierBuilder() builder.add_root_certificate(certificate) @@ -144,7 +144,7 @@ class TimestampingEnricher(Enricher): def load_tst_certs(self, tsp_response: TimeStampResponse): signed_data: SignedData = tsp_response.signed_data - certs = signed_data.certificates + return [x509.load_der_x509_certificate(c) for c in signed_data.certificates] def download_certificate(self, tsp_response: TimeStampResponse) -> list[Media]: @@ -154,10 +154,11 @@ class TimestampingEnricher(Enricher): cert_chain = [] - for cert in path: + for cert in certificates: cert_fn = os.path.join(self.tmp_dir, f"{str(cert.serial_number)[:20]}.crt") + print(cert_fn) with open(cert_fn, "wb") as f: - f.write(cert.dump()) - cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.native["common_name"])) + f.write(cert.public_bytes(encoding=serialization.Encoding.PEM)) + cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME)[0].value)) return cert_chain diff --git a/tests/enrichers/test_timestamping_enricher.py b/tests/enrichers/test_timestamping_enricher.py index 7d30e2f..9b96051 100644 --- a/tests/enrichers/test_timestamping_enricher.py +++ b/tests/enrichers/test_timestamping_enricher.py @@ -20,7 +20,10 @@ def test_sign_data(setup_module): result: TimeStampResponse = tsp.sign_data(tsa_url, data) assert isinstance(result, TimeStampResponse) - + cert_chain = tsp.download_certificate(result) + + assert len(cert_chain) == 2 + try: valid_root = tsp.verify_signed(result, data) assert valid_root.subject == "CN=Entrust Root Certification Authority - G2, OU=(c) 2009 Entrust, Inc. - for authorized use only, OU=See www.entrust.net/legal-terms, O=Entrust, Inc., C=" @@ -28,7 +31,6 @@ def test_sign_data(setup_module): pytest.fail(f"Verification failed: {e}") # test downloading the cert - cert_chain = tsp.download_and_verify_certificate(result) def test_tsp_enricher_download_syndication(setup_module, digicert): tsp: TimestampingEnricher = setup_module("timestamping_enricher") From a0869bb3b2020defb91c02022a0a84946dd8604b Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Mon, 3 Mar 2025 10:28:30 +0000 Subject: [PATCH 07/20] Fixed up timestamp verifying - waiting on issue with rfc-client to be fixed Ref: https://github.com/trailofbits/rfc3161-client/issues/104#issuecomment-2693890607 --- .../timestamping_enricher.py | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index c138cee..a779426 100644 --- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -69,10 +69,16 @@ class TimestampingEnricher(Enricher): try: message = bytes(data_to_sign, encoding='utf8') signed: TimeStampResponse = self.sign_data(tsa_url, message) - # fail if there's any issue with the certificates, uses certifi list of trusted CAs - self.verify_signed(signed, message) - # download and verify timestamping certificate - cert_chain = self.download_certificate(signed) + + # fail if there's any issue with the certificates, uses certifi list of trusted CAs or the user-defined `cert_authorities` + root_cert = self.verify_signed(signed, message) + + if not root_cert: + raise ValueError(f"No valid root certificate found for {tsa_url=}. Are you sure it's a trusted TSA? Or define an alternative trusted root with `cert_authorities`.") + + # save the timestamping certificate + cert_chain = self.save_certificate(signed) + # continue with saving the timestamp token tst_fn = os.path.join(self.tmp_dir, f"timestamp_token_{slugify(tsa_url)}") with open(tst_fn, "wb") as f: @@ -106,11 +112,18 @@ class TimestampingEnricher(Enricher): if not cert_authorities: raise ValueError(f"No trusted roots found in {trusted_root_path}.") + timestamp_certs = self.tst_certs(timestamp_response) + intermediate_certs = [] + for i, cert in enumerate(timestamp_certs): # cannot use list comprehension, it's a set + intermediate_certs.append(cert) for certificate in cert_authorities: builder = VerifierBuilder() builder.add_root_certificate(certificate) + for intermediate_cert in intermediate_certs: + builder.add_intermediate_certificate(intermediate_cert) + verifier = builder.build() try: verifier.verify(timestamp_response, signature) @@ -118,8 +131,8 @@ class TimestampingEnricher(Enricher): except Rfc3161VerificationError as e: logger.debug(f"Unable to verify Timestamp with CA {certificate.subject}: {e}") continue - - return False + + return None def sign_data(self, tsa_url: str, bytes_data: bytes) -> TimeStampResponse: # see https://github.com/sigstore/sigstore-python/blob/99948d5b80525a5a104e904ffea58169dc6e0629/sigstore/_internal/timestamp.py#L84-L121 @@ -142,12 +155,12 @@ class TimestampingEnricher(Enricher): raise return timestamp_response - def load_tst_certs(self, tsp_response: TimeStampResponse): + def tst_certs(self, tsp_response: TimeStampResponse): signed_data: SignedData = tsp_response.signed_data return [x509.load_der_x509_certificate(c) for c in signed_data.certificates] - def download_certificate(self, tsp_response: TimeStampResponse) -> list[Media]: + def save_certificate(self, tsp_response: TimeStampResponse) -> list[Media]: # returns the leaf certificate URL, fails if not set certificates = self.load_tst_certs(tsp_response) From 3f6acc09179a89b6d2c4bdf4c26e8c96d5b4243f Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Tue, 11 Mar 2025 10:04:46 +0000 Subject: [PATCH 08/20] fully working timestamping enricher --- .../timestamping_enricher.py | 22 ++++++++++++++++--- tests/enrichers/test_timestamping_enricher.py | 15 +++++-------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index a779426..b29d338 100644 --- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -2,8 +2,10 @@ import os from loguru import logger from importlib.metadata import version +import hashlib import requests + from rfc3161_client import ( TimestampRequestBuilder, TimeStampResponse, @@ -20,6 +22,8 @@ from auto_archiver.core import Enricher from auto_archiver.core import Metadata, Media from auto_archiver.version import __version__ + + class TimestampingEnricher(Enricher): """ Uses several RFC3161 Time Stamp Authorities to generate a timestamp token that will be preserved. This can be used to prove that a certain file existed at a certain time, useful for legal purposes, for example, to prove that a certain file was not tampered with after a certain date. @@ -98,7 +102,7 @@ class TimestampingEnricher(Enricher): else: logger.warning(f"No successful timestamps for {url=}") - def verify_signed(self, timestamp_response: TimeStampResponse, signature: bytes) -> None: + def verify_signed(self, timestamp_response: TimeStampResponse, message: bytes) -> None: """ Verify a Signed Timestamp using the TSA provided by the Trusted Root. """ @@ -117,6 +121,16 @@ class TimestampingEnricher(Enricher): for i, cert in enumerate(timestamp_certs): # cannot use list comprehension, it's a set intermediate_certs.append(cert) + + message_hash = None + hash_algorithm = timestamp_response.tst_info.message_imprint.hash_algorithm + if hash_algorithm == x509.ObjectIdentifier(value="2.16.840.1.101.3.4.2.3"): + message_hash = hashlib.sha512(message).digest() + elif hash_algorithm == x509.ObjectIdentifier(value="2.16.840.1.101.3.4.2.1"): + message_hash = hashlib.sha256(message).digest() + else: + raise ValueError(f"Unsupported hash algorithm: {hash_algorithm}") + for certificate in cert_authorities: builder = VerifierBuilder() builder.add_root_certificate(certificate) @@ -125,8 +139,10 @@ class TimestampingEnricher(Enricher): builder.add_intermediate_certificate(intermediate_cert) verifier = builder.build() + + try: - verifier.verify(timestamp_response, signature) + verifier.verify(timestamp_response, message_hash) return certificate except Rfc3161VerificationError as e: logger.debug(f"Unable to verify Timestamp with CA {certificate.subject}: {e}") @@ -163,7 +179,7 @@ class TimestampingEnricher(Enricher): def save_certificate(self, tsp_response: TimeStampResponse) -> list[Media]: # returns the leaf certificate URL, fails if not set - certificates = self.load_tst_certs(tsp_response) + certificates = self.tst_certs(tsp_response) cert_chain = [] diff --git a/tests/enrichers/test_timestamping_enricher.py b/tests/enrichers/test_timestamping_enricher.py index 9b96051..369610b 100644 --- a/tests/enrichers/test_timestamping_enricher.py +++ b/tests/enrichers/test_timestamping_enricher.py @@ -6,6 +6,8 @@ from rfc3161_client import ( decode_timestamp_response, ) +from cryptography import x509 + @pytest.fixture def digicert(): with open("tests/data/timestamp_token_digicert_com.crt", "rb") as f: @@ -20,17 +22,12 @@ def test_sign_data(setup_module): result: TimeStampResponse = tsp.sign_data(tsa_url, data) assert isinstance(result, TimeStampResponse) - cert_chain = tsp.download_certificate(result) - - assert len(cert_chain) == 2 - - try: - valid_root = tsp.verify_signed(result, data) - assert valid_root.subject == "CN=Entrust Root Certification Authority - G2, OU=(c) 2009 Entrust, Inc. - for authorized use only, OU=See www.entrust.net/legal-terms, O=Entrust, Inc., C=" - except Exception as e: - pytest.fail(f"Verification failed: {e}") + root_cert: x509.Certificate = tsp.verify_signed(result, data) + assert root_cert.subject.rfc4514_string() == "CN=IdenTrust Commercial Root CA 1,O=IdenTrust,C=US" # test downloading the cert + cert_chain = tsp.save_certificate(result) + assert len(cert_chain) == 2 def test_tsp_enricher_download_syndication(setup_module, digicert): tsp: TimestampingEnricher = setup_module("timestamping_enricher") From 1db8be91dbfe6d0b5913b1f6437a97190b9e4480 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Tue, 11 Mar 2025 11:08:52 +0000 Subject: [PATCH 09/20] Improved unit tests for timestamping --- .../timestamping_enricher.py | 44 +++++++++---- tests/data/timestamp_token_digicert_com.crt | Bin 5964 -> 0 bytes tests/data/timestamping/intermediate.crt | 44 +++++++++++++ tests/data/timestamping/leaf.crt | 39 +++++++++++ .../timestamping/rfc3161-client-issue-104.tsr | Bin 0 -> 4775 bytes tests/data/timestamping/root.crt | 31 +++++++++ .../data/timestamping/timestamp_response.tsr | Bin 0 -> 4776 bytes tests/enrichers/test_timestamping_enricher.py | 61 +++++++++++------- 8 files changed, 184 insertions(+), 35 deletions(-) delete mode 100644 tests/data/timestamp_token_digicert_com.crt create mode 100644 tests/data/timestamping/intermediate.crt create mode 100644 tests/data/timestamping/leaf.crt create mode 100644 tests/data/timestamping/rfc3161-client-issue-104.tsr create mode 100644 tests/data/timestamping/root.crt create mode 100644 tests/data/timestamping/timestamp_response.tsr diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index b29d338..9742587 100644 --- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -81,7 +81,7 @@ class TimestampingEnricher(Enricher): raise ValueError(f"No valid root certificate found for {tsa_url=}. Are you sure it's a trusted TSA? Or define an alternative trusted root with `cert_authorities`.") # save the timestamping certificate - cert_chain = self.save_certificate(signed) + cert_chain = self.save_certificate(signed, root_cert) # continue with saving the timestamp token tst_fn = os.path.join(self.tmp_dir, f"timestamp_token_{slugify(tsa_url)}") @@ -102,9 +102,19 @@ class TimestampingEnricher(Enricher): else: logger.warning(f"No successful timestamps for {url=}") - def verify_signed(self, timestamp_response: TimeStampResponse, message: bytes) -> None: + def verify_signed(self, timestamp_response: TimeStampResponse, message: bytes) -> x509.Certificate: """ - Verify a Signed Timestamp using the TSA provided by the Trusted Root. + Verify a Signed Timestamp Response is trusted by a known Certificate Authority. + + Args: + timestamp_response (TimeStampResponse): The signed timestamp response. + message (bytes): The message that was timestamped. + + Returns: + x509.Certificate: A valid root certificate that was used to sign the timestamp response, or None + + Raises: + ValueError: If no valid root certificate was found in the trusted root store. """ trusted_root_path = self.cert_authorities or certifi.where() @@ -148,7 +158,7 @@ class TimestampingEnricher(Enricher): logger.debug(f"Unable to verify Timestamp with CA {certificate.subject}: {e}") continue - return None + raise ValueError(f"No valid root certificate found in {trusted_root_path}.") def sign_data(self, tsa_url: str, bytes_data: bytes) -> TimeStampResponse: # see https://github.com/sigstore/sigstore-python/blob/99948d5b80525a5a104e904ffea58169dc6e0629/sigstore/_internal/timestamp.py#L84-L121 @@ -173,19 +183,31 @@ class TimestampingEnricher(Enricher): def tst_certs(self, tsp_response: TimeStampResponse): signed_data: SignedData = tsp_response.signed_data - return [x509.load_der_x509_certificate(c) for c in signed_data.certificates] + certs = [x509.load_der_x509_certificate(c) for c in signed_data.certificates] + # reorder the certs to be in the correct order + ordered_certs = [] + while(len(ordered_certs) < len(certs)): + if len(ordered_certs) == 0: + for cert in certs: + if not [c for c in certs if c.subject == cert.issuer]: + ordered_certs.append(cert) + break + else: + for cert in certs: + if cert.issuer == ordered_certs[-1].subject: + ordered_certs.append(cert) + break + return ordered_certs - - def save_certificate(self, tsp_response: TimeStampResponse) -> list[Media]: + def save_certificate(self, tsp_response: TimeStampResponse, verified_root_cert: x509.Certificate) -> list[Media]: # returns the leaf certificate URL, fails if not set - certificates = self.tst_certs(tsp_response) + certificates = self.tst_certs(tsp_response) + [verified_root_cert] cert_chain = [] - for cert in certificates: - cert_fn = os.path.join(self.tmp_dir, f"{str(cert.serial_number)[:20]}.crt") - print(cert_fn) + for i, cert in enumerate(certificates): + cert_fn = os.path.join(self.tmp_dir, f"{i+1} – {str(cert.serial_number)[:20]}.crt") with open(cert_fn, "wb") as f: f.write(cert.public_bytes(encoding=serialization.Encoding.PEM)) cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME)[0].value)) diff --git a/tests/data/timestamp_token_digicert_com.crt b/tests/data/timestamp_token_digicert_com.crt deleted file mode 100644 index 592edf4ae0e96057379dd362f8671e6385a8a8d4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5964 zcmchbc|6qX+sDmfmwn4Rb{flf#!l9;hGgGDvea0z%tEqGvZPX3LSZHfkx)@~4$2nU zMI>9Ihzg0GQ8`EFcb?~*-|Kn3&ipmkeP8!=U-!)Y`M$r`1rWH5sAvzw8pW>d15tyC z1TIy8z@-QVK@d#9J}TN{u|^5==&y^olRim-$?-=fQ9#-jr?yx2w)}(@;Dus z00vRid1G=urwSFN0aZ?Zvktir-)vnGI^U|hgZe_oyb(ptsNUAfn`p}g* z3VUGR9v0J4)jMd>Bk5>YcQH0(Z4tq37HJVO7#4Fj{Y?DK(wFt2o_j3kjEW_TD8{e`N#EwGiS>A+keWFD3 z;JQ=t^(L2%axUv|;v(RX4|)Up;Dz?dz7HLQ3##gbL|?KZ5Ai6v3U$CB3E_c{4<15e zaKj-in=K`&W?2g=7?*^E;SCp&0ess=j-F&Kml;>SYkUC13*Vb1E&H}Qc^=` z0NBnr47wlS0%)kHPDC?+$nNxel`pvCAp+$FIHTCBI7j@<&dFh<@R@SB3*NA#?5F+! z|BeGAloQIHVW|ZW(>~+iC<-Pe;K|Ik3aNL;|gjxX>fH~2OXcB9D5{nH`laq4~ z@|BnMAZyZ{tVvmSe?PfhA^en~p~6oQ{?OoF38Hjl$;*N0%ezMbfuPZ6f#@hebaw;* z#d81L$=@9l@XJns{qJ7jSkv8n+;$Sd{NniEri9%qSn|V&ELaeQ#$5qHe1_AMN(`&v z2=yU!4udugcB%bJT(O+!fEpp>v|2Sm%bFeh$+9}(fb|@$?S&D; z6bt{a-N8#q+-7gNtkzn5hHb@YWjp=qsOzr->a$U<94Xgvh16p2qOB3}`>kv)ybUWI zJ<~T2B{oKw7oi`t>`Q%PX;|I4E)!foQR^{7*Y(jF6jT1xIK856KCS25e$THV&pxJg z+029LJzj)8zUX|qs$m^St; zZlw625RDsTv)R`H8tzjPn(np*mkk?1ac`*=uv^#gD5%!8=(32>$W|BUI}| zhJoyA0P5irYW|>`z8zX^fs^|%9>f*?_7EGeNi~}Da8t2b9+_VyWPYXnd@$O_P4RL1ukHG+&SY@UcgnpdtU z7hba<4fdXWo^5*5k67AycEj;A3rUYd*EKJG_2ifE)f)-)ky(P-O!9(9*yzPEX@2W= z1>o>4)PtB8EFUms_us>#V$KV5Pb|F9Xy%G6+7H*%9XoO9t%b(&pl$!0q|8EJ*+bl% zAeHCE6sf+EMfAkTQCKpM6qgJ+Z`i%iVYSb_;)((!Qeuv%Ce39$4-ChM<xII=|vIv8EHgSl==cCdh zwc36s4WoN9r^4WVmRjxrFh&4cZ_W86lO>Q`=Yt|HYQQ|_;BPilT{!j@|Z(Bc=1oDFcVkLB9a zf+Q&zW31mln|FTd05dgh3HrgTq3)lcn6U%w}xefYq~l?#(AhoL`$rZ3dj*cSj2S72Vj)Pd%@B z?CCK5%u%Nyzql_kFfj$hiLMIPH2>x!rpNgiXb1(IwS~6b9GHtBVXINz(aSz$c^bYV*cH8}Gs$F)M{ za_9MPlGX{!1Lp(ro-aC=gK|Z_G)NHiN-`vPi$n#)8!N{TEPT!`JDm}oYa3fcr{6Zw ze33YpzPWijE9hSHlg38xz>!wADB9+VoUXVy(Q5Nto=V)^9t?jLPFX-m29G2WBeg%)7f`t6audfiTg~e zV2W;o%xksOL++a1-E2-h^N!9{UNuf9BvfVo#^g(pEq=+qCmU}*q`Pjyha*-DvT)J@ z64K(M{v2ln_9rSmO75Rs;mUAm4_H>NPp6oGkzf5bkDcHoc~!l%X;71jElnrF-=*qQ zsts_YBOV;5WumK%9%0+A&wIXVH&O%rLKsI*KogcJx}2LfdPE{A=%?{u@sVXuyzXtO zI9fyaY$Q+KIi|1aX`gMZ;et_L8dezLGlQFja(Wzx=GYkdN0n80Nbt|^2E)Dr1j>tK zV&Q%gi}5=5rHvk$xuww~uW7Wo9rqH8fn0HU{(p%j3JCuo7Vq!G(!pSGLF7`*SP%9c zT0d(pB%ngBMSrik5C{ah2>V%c{Ttx?J7(3-UdEgpuby^3bPx~*(DI8-`C_tdLK-XQ zhaH+GrSdiGi}0CUC1-eJ_B8{7tGszCoC~(C*Zla4%1^Jfv-D284pJVpWU>@&&E~ij zQI}I_$LU}bbnj$xKvQ`YF2fsxOnJ_%$1FQRSf!4c#j5)SzLsI&eGeOGgYmmd$vj(h zIs7zvUC8aa!UAQh%c2uc%|csuSFwV&Ygwq*`{1WW1zvV`Io9@<7~qxW`M`+n`}Q-= zQ+M0UU1k*L5FIa)UB)gYMZKuFlCFLE^lu?gD$1H4dULBlm@f|ZE9c2lhR1&*Xfj_3 zD6%2wAcsbwm;FQOrY>hFB zX??MTicG!z@Q8+#2fc273-Ti^6DClK&Tzie-;&mZ%F9_Qt0PwiE(A#H9J^EI&aIcq zgFhbeIjVH?jmK^I=Lyu4wUx7|Jnl!ggF%x_DGWmV<Q=FFjqL6Dd%)-aNtSC0oylGJ1H4FBYB+$~3 zbh9jiPCVn&G+Hj5U*^+DVLAT{?=G{{$jnmuUiAN979{{7Giwj9{*p&~fu+B2^WMj+ z9R!4oMPh!wP}Zv7Q>Om_*6Cf!klEx1`20gQp>|i1*`&P7l>e*x`6)(Yy&Wljuf4E< z{tr570%SfB$$W}c*p2kR(&<0L_yR0DVPH@?fJ}8NDETYJpPnud0tiGAK&(d{wV6|K zCFWvzH7YoV*6Qu3Ycq>Uang(DTnc%rh1cVF(AL85^5TR2YI)c{vK8z0w-fzZsva+m(@=P14FD4~V4Ad=C8jE*exd@2ZA zu!LU!T;v;vg5`QS`rGp>3I`Y0AWz~7dQG+WD;jvCtxF$~Y%eY54QRY68{6U={uu0d z)cS(|66uJ>S3K>>m>mE1QP@SrV|-vp2duQ%p2(=}-jXidSnlx2#;04N$QS?2Uqt3( zM4-2bkMK9vN7gbD5yNUD%N22HTTai97IXh*mV>DT73SF8aglueI0Zp~-~j?8h}@L} zg8!8c*I)P9_H+)3(a!;%-#c%#5X8P;8i@ekA9IulCUU!MZ{rVu?{Nk*Aou_tfZSn# zvXAzidw+biSM$vD=Rl()aiRmUPWph{AEBTKVJK`W|4W*MqWr5OfR0&d{dGG<>OF6@ zG?JcbLDlH64+j^Zyf?ZWLJ|-M$cTZ$@c5j96?Zv76up`VP%>zX+aAJ&0^EuXn(E}+b8DD@i0ZWDPUCdBs(J~MQhd+Ax@f+ zeoj1KD^F8~$2q--khCbX zMrY_MN{P)ZmpjvcIn>GqIv#Eit}D>%b`x#i`*HXi&xf0twq@tX;5gKJrvD=YiX)&| ziMq;;49dGUxD3bIPwmJqo`c7-n^qUEN|tflsgr2$T}s(*a<#-fD2eLa$|l_{UuWn= z80DtHksM2{OWWMudbNyGc4J@_gkzn=M|37!xzdEXRfFAi;J0zJypG(%;WUizHIrS@ zQz;^lX!Xr)i=ztm8~FW??D!gC^4$xcuBmy-Za;Ee*u3s!D6?LUPK_#ovt3I`1&Pr#=VevwBV|WgT zzZ2(huQlE`;H1X22~W`^U{<>=BYRV@rtnj%6CjzVEV~Y+1%&%680H#_G~GR zC6r`MLS^5V$l;mkS3S;o`n`I+p6C4exxe>vUElk@uJ?5TB-R`N0tQh5B-T_Iop^*! z#1<2X2AoA=jUf+3fWSP+|f=j01x} zFb2xRh;0NFj0pmqJ-8A=O$Aj+`}L)8z0;b%UDB%yeY1*qWZLH^zcCgKPrTg{;c`8b zFC^45lyX6(AHpD-YLdoJG2)iT-=On z`sn;927Yw~#6WgU6UhM3!5|h;7#rBa49NOj~ z9N|!%`Jg*t2+ zSIbTmJP8e&c;HZtPQZKWw%+%RH=j6d`e{|XN_dQ0t$MoCPIxa@hiIEV66GELUd?qC z+n&CR5X&2<#VPC!&)&k2bt*#cr@s2Q|IJg3k2}Ce+2;_H1IjKC zW{-GaT3o{bvo1nFO)bKWGWSbeq z6J4w7mpk5I(RI^^u8n3~1CPUnVsy!;l>KUgD{=C^O zy{lX5tArI-4bgC5$jt_w3+G16|GpB+#K$J5`dRVS#i!UsZ>8$S_x$G?I3umm<;6TL zNp~&d?_I02Tl0+R8eS92^|5raX*SG|S>046j6PrZlmHsw8#58o5^;{Vo!+K3i#0_v(+yA?LFzbnZ4&;H+fe!n@*O@>wkcc zF7l_;l3uDFe$*KFlzDYK){QEO5j{7T%QF*X{)Q{VeT;yUI+8H{g3nJi3v_|co#m8` zK|oZlr7gWu8MMuyzj@|xg5W#f-Q_Cn$Vm?wM6U0u&4->H?D$rrvO`*1Y-bvszBI~F?Z}c`rzu{gB)(^+F;AV+4Mj0}K6N`ICA z6jFaix&o(lO`_JK)4CQk86nmw=cZv^@zMQB}C zehLK6a7LNNHWxxmgq9E8ACN3e&lMa>zqs9cEx$b3ic8<+gj_e*4dptOB`4I(=q;wx z!JIlf!Ze`5y6C4265eHYES+72jOC9e$~s^wQ+IlMm}erCRo7pVcM zb3OX`Vhi2)h549?dh{BN*I5bvEvJX~b2oBnCa>)#GQKVk-BpI2X_}d53S6-`Iq zZ4!?^Ue>QggOXR`#O5cD7&AyP+w1k3jk{wSlQ(fggXaU`X+g#|&#LVL3QS?I`W0KN zEf+dtm18wio@Ec$?buEf6JTw{20 z&3RVYehU*;d-sY}2bwK)mEq)WtEaE{UO7UJpQi# z!Ah9T4Y6Ty+QYZD$_smstCTJdRO9NMY)}t<m0SZ_|>X@G;2S7>D7)RuPux&wutEq1p7PFWe)S&k9xcMJ9P<~{5YfxN*LrEY7vN-ks5e`EB z$bFD;Ao2j+KT0`b?KiK%V1QW4NGvjU!{OoI%2ahSzQ1A;?&gSxJ7KY&a2y`*i*<(M zJmHQQjF*qIqbCLnzl0;W!M|r0(I0PN@&&YT7cXB=XSgE){y!pGkMbA!q3VDtpq!aYFA7j{dyDn#z69J@aPe-`-5CB;(hd9Jk1 zi>58I%I|vvtO63811u&(=I2?a@k+u@`EO)awPHgkeSTCOPFs#RZ0Q znl(Oh!!JAH%X5;6#<9U}qN9&x+U>n&2iJt9w&}BjEJV+t%M2juT$K-*SpaQ6&LSoI zN3-(pg29(N(95%%e361`7#x#jzKucfQ2%Kyr z%AOWzRhDOKw4BV>aqrff^Vv*m8zo(WQ$bn-8Vri@B5J44Gw;tN8%V>hWat}ERa14u zYK1u_M!Wm7c@4{?uKRe!)7L0|%rYx1(C`z8u|^9Ox2cFM9u2O;P3uN9be;aFPjm-l zrI}q5;a3UHUF@uPTkg3HldJ>r(q#-!|1_!ExCuocof^E7Gw3{cYnmoTCkh&2zaW0+ z276QG)jG3ReJ;wZd`I(lRgd*M@`j+wyr*@MBnUr1g7A>rXTa(o!sU-U-2+X07BL0j z{@(l2L6GD+`9Mz(@c#Q7HIkX!zaDI{0f_^X!Hh^=fEyr}4dC-Cuc$d#J{(KA_bvv`SiOlB4<;OC+0_jX(N+m==w)!gajmAC4@XgJPgcn2a6SQ zLklH=sVa?z>k9GOnuHwQL^N(#PrxCO`$(7A*ob+5cQY`v%y#(YS{O2irf~XVma^$) zPbRUcfgzxv^;f4TMk}LUwB=FFvu1dfShK!QOILG#?QCG#Xd(za7d@o61T((w!x)b@ zxj#1r&ln7pQnktFs2R%9soa$D*tKhSGHHK(VrUN$8d^?Yd#?KhD{l6XfGt?P+ll%H^D{;>T} zl4Zg>eD8)>s!O5fxCQ#0TXVdxwiZOPpBCVd{!g;TL65U!kUl7rGdyac<6x4$< z5>HROmDFeMj@BbRqHimYRl9;kPb}OS64H2EagvSOzIpzk21+Dx;ciwq-~KMbVC(Vo zCKdJGZPkxj>Lyf2!UI^Q->C_tUb)?SXI?i3-A2dmJ^5H)#w2qF=(wWDy4AIVlQzWr zeKa;O_#7H0JH{QS>&za1d~dq=b9TX4%-L>!=wj*OMf~dz7hCVIXq_}NFzHE`v~4$L zXGu$uy5v0bX5B^oy7T*rHNmq;qeI9HND2%oC?N`nNC?s(ijopa zsDxr5?Lj~Z=>}2mj6d~q&h_{BJolbIzx&;>*1PvwpS1uIOCA6LgQx%!%MBQvc$8k$ z1`~(|oK0f6L>`I(gCJ-)zyzanjM4*PAy6=gnhFRjfH8cZNyiL=W{0Ih!>$5hNnj8N z#z2`EwIxXfV}bz3_pXFcQ$f`~pUbDp^2}LPzC%1i*NIX5Glct%$47z0h~O()mjp^y z*BF?_mma_G#;-E9HEE91nm^GV=@i!;Z5i8yu?TdzfSJ`n*CM)yLA2$*WeGsPbS@`c z3XqZoq|sCl}=y1JRx zzNYh~Soqx+5Chq_V`v6|4hFG=!dStU7HAfLnIh;}>GWN&9w)qg2t+hLfS`;)Sh>Dw zeQm@EoI934baMB?d7MUQX&|J5!yHUfGH3v;fL4@IkO6GSQVu|)NdN7H6uR|4oq~_B2Wk~5EZ+zGY zDpfsrQsSN>NFma3q9tDdAPA-9u^){MADXOeXQ7|KRWuK4-Ab3e^_f-bZiuKYE9)Vd z)F{WAth<%h>I&;#T6qkZ_=7${=+CkRZEc*Zx`WNk)5-`u%gq;|jw!ZU69aRt#71%N zrl+JA9*^w%?9Q$CMs@87itPdJfulP-?V5Ro@%3WBuk`EC8Py!{33Q;&f;@N^R$2$w zrjavF4Z*s!R2a(FibSs#gwhc%OoxT*G6#PFPsktw5sBH)L+qHPoUSnwcIO6stW{#7 z%vW;N`#b6C;%~F@t|r=8Z1Sx>*+$nZwI4o{6+?eG!^EtsQDdFTV!|24?&n}SR5qAl zHj{L?uOX~+=b4FciGZ|+)8wRtih*Iy(J?+plVj%_Qh6qVbi5j96D%kCdu=bQ#=P}V zcJYm78hJcm#ld1KJvCK6h*tJ8&E+b}&uS%VY`@AhPd=3ycqHhPLZ(E6wh+pZ<#XfZ z$O_TI$031ZcO7dKk_jIA?RR{VPK+s;Z!U{h3%=vlteNUOCHOT@FU&4uD8?)4gXZaF z?6ZtTNwNGx^E$e|sp-RkwduKH>z~qhha(JKk-Qo`_z)S%X=eB?a7@!^ znAKIN4y62Pj&Iq8R~t7%dV8qNb*S&;ntRKLiK_^#{%azz+?8q6FQD zL@y;tNsOnvJJuV6bHbxMF$6CZjvOyUO1z-RUjVJ2E5K3m-&goL;c+fbM4YF`Unfgq zu--(1BymqrNu#{vF#s>cJ^?5va3GYeLuyej=!y7X(u1Jr*!>MP{P-q1AcQgj0rdlX z0a_UBbQmLu>?hOSoCE@F_euz01Q`4<4GcPQ<_J)AP_;_P~VHeI7hjAie36fr( zcpS!Gl58l>!v*VyawEFq0Ue3~yihej1yIgb%$ARm{jIOw_`i9I@x}u@lzsL=4?x-Y zLLDUUnwQivz-&y69S60Y<_>`N20S@ukpmt?WqznDuR76!+CMl2A>m=J%GCI0%Fq^E z?qF=4hHu_zqh;4w_j=Yk=u-AV>SCt8<`?;02=^A^ipgzI0W>ZiYUdtN!TwBCHAbk^`A zVtAf6t&a3cWB>i8fL8eOR)QN<3ZufOPk9HY15b=_W#ZovaVYlW(U&~F8rh&TJa}e# zTOqd{eOH1bYEZcs3v27~6J%+NY=o0zx_JbVp z?Y^sAn@WTIHMiLAtMSoz1V*UKh=z5P6`tZ*j>!pbZ?T(hat8A>avZlyY&w3Ed3s-N z)znt4{q}{G6Jx5>jzxX0C1uae?)nw2<-F_$NHjG7i8}iy=0dI}vey@})cld6qCGqR zwW(9q9_4SC571%sa2@KvN~fI%VCot!nV%+`W>quMUo@7sTxtSHK4%XNw$zo}~$_ z$|z2Nz?m*`<_V9Aprrzf`|k8f6lLV`4`iI(YQI!anQF~t=qf7voGV_fUVXt?ZhH7C zlTr|;-nJkOsHi@+l>zBhaf-RKtBA4k{#Zo^Onu^J&kOi;l$!ecw5HuM2Cl3IqjhfC zN)4KP5;87+DMyW7HL9AayR>v&1vbZ^j?yHGeST&*g?fm5RioSTeB_bavKahK9T%w) zt#>)Ld%lHk^vrDBScAd}jpuPB?}qceJ9%q)G~<_ct}(u?4B1hG9c`YTWeQlbJkrhV`fa@3{HzfZ-}^k{>&d2w z)LMHMnO;jX76<%;Wk;F~E%m|F=hm%ncwRzG)#EsAcHJLHvlB|#S*jh3SQr1yG_THo z@?yl$JhbJdWcy{kti>umD=Qs)I{{~`wPM5dh0*3S`0+T-K^+4bV1Rtym z0_TBn!eBhTU7S2HSj2f8(GBr4yNLXH3$t&aMYwwUc(@>(h=~6Y(FT;i$P3j1Gyt`1 zm2BlG#ot{U8OXo5w7+8dm*og6kOd0IJoTmA5zAw*liALmudbZ5D4 zbe=VDkx~25<8SSs?BZ`Z9y~kCJW&9uL`Jv6CWfDM`_-%+us(7~a&pHbb*k&&cENN1 zghe6oX&-WzB(5cd8s>>9FSS;=yPThVBf5*LS}aVbCyTXklxxLOKfCgL(TzW6PVMU8 zurFU+OxMTxz3?r9`|N%UN0#%Y@C((&E*3A<##8IM_9UFCV{b6zs{Ej#$CBdIbsHOw?2$W3JUq4YBc&zb%eLDUv~PN1d8Nreg{h!$7XJvcL<>&scJ z>TrKX@qG~ZLWjcQ%=#Df10?nMr>q0De($ydn&K`6q8>KCUzoN|=T0oMF}9yMQM@kI zQ^c$h-hAnChEk@GccZ~bbi_jaye_J$#6J0oE1yb|#CdJRRj%N9=Q$R@O?p_$LB&`m`BTl7uvs?BTo9sYX(;@JvG^sv4?} z1f5W)Yq5AgR?k7{^i^-qB>GyF$!v?FLTz8ZI2#3lk|*jy^Bh6-xGDXp#xA8v!!SG` zBLzQw4G~Fn>0)EK-BRv0NU{kaNL4Vjele?Fy8>0TOS#R;?wW}Hn4@bd{#^GM4L&0(tUQ_yL5`-5ZK@O7pXTbU&!sV|!-924=c32v~ z{j>L_gP_TE@}8z1IP~vt)Mz-lf8E<-1IRs-!Hnob05?D`%b{%aiO5sO*MuBzLoiPZ z=mczl$45f|$$xJJMe{=u5-K;c9`iG>UsyMo)Fbw_oAk!hC55ht#z*e=SBp)Q-dn7Y z8(Js{Oyv+08ig7%3^LEbnto`PTTL~UQBGQny(o)6DevieJ>r1Rxrj2axvd?Ed z^*(}JIF_yc=A;nqm>TlXNa^v|L_CQQ&9+yF0Qb{snV9rc@;1!b48TG;$;~tym)7Dep_; zcJspvbDW5jfcC8oofma-&{6Mw9T^$s<#|W^qk2B*Q2(LAa^1&EMM>%sC+GbKRX&m& z8>hpCbsdP2xevj7#}!wK9eHIkniurPj=363OkEzzx^1_5c!-m$jh*#E$GiTXV9Cld z(>I$UUT}l!GWDLq!WhK{LW6lb8`c=MImsb1YrGqIs@$M>Z4+f0g!V zo3p=2WX)Nvr)#2Ck7!{lkdYm8rU>NMZ2HSi8A-?#_OieMxL(QGxIFceo5K4shyb(M za4}&odlH}a`tURNw89}OeOlSx2l@zl^?l+k#n&`%DXiPnZ3!>Dwo$gE9 TimeStampResponse: + with open("tests/data/timestamping/timestamp_response.tsr", "rb") as f: + return decode_timestamp_response(f.read()) + +@pytest.fixture +def wrong_order_timestamp_response() -> TimeStampResponse: + with open("tests/data/timestamping/rfc3161-client-issue-104.tsr", "rb") as f: + return decode_timestamp_response(f.read()) @pytest.mark.download -def test_sign_data(setup_module): +def test_download_tsr(setup_module): tsa_url = "http://timestamp.identrust.com" tsp: TimestampingEnricher = setup_module("timestamping_enricher") @@ -22,27 +26,36 @@ def test_sign_data(setup_module): result: TimeStampResponse = tsp.sign_data(tsa_url, data) assert isinstance(result, TimeStampResponse) - root_cert: x509.Certificate = tsp.verify_signed(result, data) - assert root_cert.subject.rfc4514_string() == "CN=IdenTrust Commercial Root CA 1,O=IdenTrust,C=US" + verified_root_cert = tsp.verify_signed(result, data) + assert verified_root_cert.subject.rfc4514_string() == "CN=IdenTrust Commercial Root CA 1,O=IdenTrust,C=US" # test downloading the cert - cert_chain = tsp.save_certificate(result) - assert len(cert_chain) == 2 - -def test_tsp_enricher_download_syndication(setup_module, digicert): - tsp: TimestampingEnricher = setup_module("timestamping_enricher") - - cert_chain = tsp.download_and_verify_certificate(digicert) + cert_chain = tsp.save_certificate(result, verified_root_cert) assert len(cert_chain) == 3 - assert cert_chain[0].filename == f"{tsp.tmp_dir}/74515005589773707779.crt" - assert cert_chain[1].filename == f"{tsp.tmp_dir}/95861100433808324400.crt" - assert cert_chain[2].filename == f"{tsp.tmp_dir}/15527051335772373346.crt" - -def test_tst_cert_valid(setup_module, digicert): +def test_verify_save(setup_module, timestamp_response): tsp: TimestampingEnricher = setup_module("timestamping_enricher") - - try: - tsp.verify_signed(digicert, b"4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef") - except Exception as e: - pytest.fail(f"Verification failed: {e}") \ No newline at end of file + + verified_root_cert = tsp.verify_signed(timestamp_response, b"4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef") + assert verified_root_cert.subject.rfc4514_string() == "CN=IdenTrust Commercial Root CA 1,O=IdenTrust,C=US" + + cert_chain = tsp.save_certificate(timestamp_response, verified_root_cert) + assert len(cert_chain) == 3 + + assert cert_chain[0].filename == f"{tsp.tmp_dir}/1 – 85078371663472981624.crt" + assert cert_chain[1].filename == f"{tsp.tmp_dir}/2 – 85078758028491331763.crt" + assert cert_chain[2].filename == f"{tsp.tmp_dir}/3 – 13298821034946342390.crt" + + +def test_order_crt_correctly(setup_module, wrong_order_timestamp_response): + # reference: https://github.com/trailofbits/rfc3161-client/issues/104#issuecomment-2711244010 + tsp: TimestampingEnricher = setup_module("timestamping_enricher") + + # get the certificates, make sure the reordering is working: + + ordered_certs = tsp.tst_certs(wrong_order_timestamp_response) + assert len(ordered_certs) == 2 + assert ordered_certs[0].subject.rfc4514_string() == "CN=TrustID Timestamping CA 3,O=IdenTrust,C=US" + assert ordered_certs[1].subject.rfc4514_string() == "CN=TrustID Timestamp Authority,O=IdenTrust,C=US" + + From 2ffe124d952dfdf37cfd92c70c1e76293f83e162 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Tue, 11 Mar 2025 11:13:36 +0000 Subject: [PATCH 10/20] Add unit test for invalid digicert tsrs --- tests/enrichers/test_timestamping_enricher.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/enrichers/test_timestamping_enricher.py b/tests/enrichers/test_timestamping_enricher.py index b5c6b77..1db503c 100644 --- a/tests/enrichers/test_timestamping_enricher.py +++ b/tests/enrichers/test_timestamping_enricher.py @@ -17,6 +17,21 @@ def wrong_order_timestamp_response() -> TimeStampResponse: with open("tests/data/timestamping/rfc3161-client-issue-104.tsr", "rb") as f: return decode_timestamp_response(f.read()) + +@pytest.mark.download +def test_fails_for_digicert(setup_module): + """ + Digicert TSRs are not compliant with RFC 3161. + See https://github.com/trailofbits/rfc3161-client/issues/104#issuecomment-2621960840 + """ + tsa_url = "http://timestamp.digicert.com" + tsp: TimestampingEnricher = setup_module("timestamping_enricher") + + data = b"4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef" + with pytest.raises(ValueError) as e: + tsp.sign_data(tsa_url, data) + assert "ASN.1 parse error: ParseError" in str(e.value) + @pytest.mark.download def test_download_tsr(setup_module): tsa_url = "http://timestamp.identrust.com" From 294033f156707e7f60da1841edc04844cd826d79 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Tue, 11 Mar 2025 15:44:04 +0000 Subject: [PATCH 11/20] Fix bug ordering tsr that only have one cert + more unit tests --- .../timestamping_enricher/__manifest__.py | 32 +++-- .../timestamping_enricher.py | 53 ++++---- tests/conftest.py | 8 +- tests/data/timestamping/intermediate.crt | 44 ------- tests/data/timestamping/leaf.crt | 39 ------ tests/data/timestamping/root.crt | 31 ----- tests/data/timestamping/self_signed.tsr | Bin 0 -> 1600 bytes ...mestamp_token_http-timestamp-identrust-com | Bin 0 -> 4767 bytes ...stamp_response.tsr => valid_timestamp.tsr} | Bin tests/enrichers/test_timestamping_enricher.py | 113 ++++++++++++++++-- tests/storages/test_local_storage.py | 11 -- 11 files changed, 158 insertions(+), 173 deletions(-) delete mode 100644 tests/data/timestamping/intermediate.crt delete mode 100644 tests/data/timestamping/leaf.crt delete mode 100644 tests/data/timestamping/root.crt create mode 100644 tests/data/timestamping/self_signed.tsr create mode 100644 tests/data/timestamping/timestamp_token_http-timestamp-identrust-com rename tests/data/timestamping/{timestamp_response.tsr => valid_timestamp.tsr} (100%) diff --git a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py index e945350..2c81dbb 100644 --- a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py +++ b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py @@ -15,25 +15,21 @@ "configs": { "tsa_urls": { "default": [ - # [Adobe Approved Trust List] and [Windows Cert Store] - "http://timestamp.digicert.com", + # See https://github.com/trailofbits/rfc3161-client/issues/46 for a list of valid TSAs + # Full list of TSAs: https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710 "http://timestamp.identrust.com", - # "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping - # "https://timestamp.sectigo.com", # wait 15 seconds between each request. - - # [Adobe: European Union Trusted Lists]. - # "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request. - - # [Windows Cert Store] - "http://timestamp.globalsign.com/tsa/r6advanced1", - # [Adobe: European Union Trusted Lists] and [Windows Cert Store] - # "http://ts.quovadisglobal.com/eu", # not valid for timestamping - # "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain - # "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain - # "http://tsa.sep.bg", # self-signed certificate in certificate chain - # "http://tsa.izenpe.com", #unable to get local issuer certificate - # "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate - "http://tss.accv.es:8318/tsa", + "http://timestamp.ssl.trustwave.com", #timeouts + "http://zeitstempel.dfn.de", + "http://ts.ssl.com", + "http://tsa.izenpe.com", + "http://tsa.lex-persona.com/tsa", + "http://ca.signfiles.com/TSAServer.aspx", + "http://aloahacoin.chain-provider.com/tsa.aspx", + "http://tsa.sinpe.fi.cr/tsaHttp/", + "http://tsa.cra.ge/signserver/tsa?workerName=qtsa", + "http://tss.cnbs.gob.hn/TSS/HttpTspServer", + "http://dss.nowina.lu/pki-factory/tsa/good-tsa", + "https://freetsa.org/tsr", ], "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.", }, diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index 9742587..6f9aa56 100644 --- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -1,10 +1,11 @@ import os -from loguru import logger from importlib.metadata import version import hashlib +from slugify import slugify import requests +from loguru import logger from rfc3161_client import ( TimestampRequestBuilder, @@ -23,7 +24,6 @@ from auto_archiver.core import Metadata, Media from auto_archiver.version import __version__ - class TimestampingEnricher(Enricher): """ Uses several RFC3161 Time Stamp Authorities to generate a timestamp token that will be preserved. This can be used to prove that a certain file existed at a certain time, useful for legal purposes, for example, to prove that a certain file was not tampered with after a certain date. @@ -33,6 +33,8 @@ class TimestampingEnricher(Enricher): See https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710 for list of timestamp authorities. """ + session = None + def setup(self): self.session = requests.Session() self.session.headers.update( @@ -43,11 +45,12 @@ class TimestampingEnricher(Enricher): } ) - def __del__(self) -> None: + def cleaup(self) -> None: """ Terminates the underlying network session. """ - self.session.close() + if self.session: + self.session.close() def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() @@ -68,40 +71,47 @@ class TimestampingEnricher(Enricher): hashes_media = Media(filename=hashes_fn) timestamp_tokens = [] - from slugify import slugify for tsa_url in self.tsa_urls: try: message = bytes(data_to_sign, encoding='utf8') + + print(tsa_url) + logger.debug(f"Timestamping {url=} with {tsa_url=}") signed: TimeStampResponse = self.sign_data(tsa_url, message) # fail if there's any issue with the certificates, uses certifi list of trusted CAs or the user-defined `cert_authorities` root_cert = self.verify_signed(signed, message) - if not root_cert: - raise ValueError(f"No valid root certificate found for {tsa_url=}. Are you sure it's a trusted TSA? Or define an alternative trusted root with `cert_authorities`.") + raise ValueError(f"No valid root certificate found for {tsa_url=}. Are you sure it's a trusted TSA? Or define an alternative trusted root with `cert_authorities`. (tried: {self.cert_authorities or certifi.where()})") # save the timestamping certificate cert_chain = self.save_certificate(signed, root_cert) - # continue with saving the timestamp token - tst_fn = os.path.join(self.tmp_dir, f"timestamp_token_{slugify(tsa_url)}") - with open(tst_fn, "wb") as f: - f.write(signed) - timestamp_tokens.append(Media(filename=tst_fn).set("tsa", tsa_url).set("cert_chain", cert_chain)) + timestamp_token_path = self.save_timestamp_token(signed.time_stamp_token(), tsa_url) + timestamp_tokens.append(Media(filename=timestamp_token_path).set("tsa", tsa_url).set("cert_chain", cert_chain)) except Exception as e: logger.warning(f"Error while timestamping {url=} with {tsa_url=}: {e}") if len(timestamp_tokens): hashes_media.set("timestamp_authority_files", timestamp_tokens) hashes_media.set("certifi v", version("certifi")) - hashes_media.set("tsp_client v", version("tsp_client")) - hashes_media.set("certvalidator v", version("certvalidator")) + hashes_media.set("rfc3161-client v", version("rfc3161_client")) + hashes_media.set("cryptography v", version("cryptography")) to_enrich.add_media(hashes_media, id="timestamped_hashes") to_enrich.set("timestamped", True) logger.success(f"{len(timestamp_tokens)} timestamp tokens created for {url=}") else: logger.warning(f"No successful timestamps for {url=}") + def save_timestamp_token(self, timestamp_token: bytes, tsa_url: str) -> str: + """ + Takes a timestamp token, and saves it to a file with the TSA URL as part of the filename. + """ + tst_path = os.path.join(self.tmp_dir, f"timestamp_token_{slugify(tsa_url)}") + with open(tst_path, "wb") as f: + f.write(timestamp_token) + return tst_path + def verify_signed(self, timestamp_response: TimeStampResponse, message: bytes) -> x509.Certificate: """ Verify a Signed Timestamp Response is trusted by a known Certificate Authority. @@ -127,10 +137,7 @@ class TimestampingEnricher(Enricher): raise ValueError(f"No trusted roots found in {trusted_root_path}.") timestamp_certs = self.tst_certs(timestamp_response) - intermediate_certs = [] - for i, cert in enumerate(timestamp_certs): # cannot use list comprehension, it's a set - intermediate_certs.append(cert) - + intermediate_certs = timestamp_certs[1:-1] message_hash = None hash_algorithm = timestamp_response.tst_info.message_imprint.hash_algorithm @@ -155,10 +162,9 @@ class TimestampingEnricher(Enricher): verifier.verify(timestamp_response, message_hash) return certificate except Rfc3161VerificationError as e: - logger.debug(f"Unable to verify Timestamp with CA {certificate.subject}: {e}") continue - raise ValueError(f"No valid root certificate found in {trusted_root_path}.") + return None def sign_data(self, tsa_url: str, bytes_data: bytes) -> TimeStampResponse: # see https://github.com/sigstore/sigstore-python/blob/99948d5b80525a5a104e904ffea58169dc6e0629/sigstore/_internal/timestamp.py#L84-L121 @@ -186,15 +192,18 @@ class TimestampingEnricher(Enricher): certs = [x509.load_der_x509_certificate(c) for c in signed_data.certificates] # reorder the certs to be in the correct order ordered_certs = [] + if len(certs) == 1: + return certs + while(len(ordered_certs) < len(certs)): if len(ordered_certs) == 0: for cert in certs: - if not [c for c in certs if c.subject == cert.issuer]: + if not [c for c in certs if cert.subject == c.issuer]: ordered_certs.append(cert) break else: for cert in certs: - if cert.issuer == ordered_certs[-1].subject: + if cert.subject == ordered_certs[-1].issuer: ordered_certs.append(cert) break return ordered_certs diff --git a/tests/conftest.py b/tests/conftest.py index a94abcd..bf338b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ from typing import Dict, Tuple import hashlib import pytest -from auto_archiver.core.metadata import Metadata +from auto_archiver.core.metadata import Metadata, Media from auto_archiver.core.module import ModuleFactory # Test names inserted into this list will be run last. This is useful for expensive/costly tests @@ -139,6 +139,12 @@ def mock_binary_dependencies(mocker): mock_shutil_which.return_value = "/usr/bin/fake_binary" return mock_shutil_which +@pytest.fixture +def sample_media(tmp_path) -> Media: + """Fixture creating a Media object with temporary source file""" + src_file = tmp_path / "source.txt" + src_file.write_text("test content") + return Media(key="subdir/test.txt", filename=str(src_file)) @pytest.fixture def sample_datetime(): diff --git a/tests/data/timestamping/intermediate.crt b/tests/data/timestamping/intermediate.crt deleted file mode 100644 index a9242f9..0000000 --- a/tests/data/timestamping/intermediate.crt +++ /dev/null @@ -1,44 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIHuDCCBaCgAwIBAgIQQAF/lJAVu6kSuFeWPUTs7jANBgkqhkiG9w0BAQsFADBK -MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScwJQYDVQQDEx5JZGVu -VHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwHhcNMjIwMzE2MjEwOTA1WhcNMzMw -NjEyMjEwOTA1WjBFMQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MSIw -IAYDVQQDExlUcnVzdElEIFRpbWVzdGFtcGluZyBDQSAzMIICIjANBgkqhkiG9w0B -AQEFAAOCAg8AMIICCgKCAgEAqGL0RYFG7mL0RgSXLynLNWhEVrhsKhrVL4rSG+NA -p4v8TbAP2YXsqWB8yZgj9DQ55AECnmQ2Uo/BqQSsI/AOr9ctqZykItmca/nGjKez -l1kZS2YoNc4Zjj+7QO9iNunclA06fBhI+iQHAam7isQLK3CwXRDLzKkMs7TisMoG -QOSd0M8P6YY/QOGYv/+tCxmfvUz2GjWzQQemgiuLjvGhPwo+hrcNzays9j0G7QtA -LkJ0KfvJS+guvCvSuEfDzt3BaPIpD2q6GYK+MUiNis3uwwngauyL4r048wdvUSsf -92Kyr6T1pAfjjPyVDNazf/w/BjzA6ewNevFVLNfE0DhQkXMmsNVGBzY5Phhlp5fb -TwsrD19K0FPgbGO/l/Zp2dheeiCbe09bxbhdeahSBtTVPca4Vu3Ljz+PRZjFodq7 -+lziqqpqqCP/ikEnmK/QkxjCG7AkX384dxg7yb5jjtXOnP5Yv4SXuV4SNNVVUBJf -bLXyYAf3Q0Dal85ZxNQd0QNPQIsYWv9ttTMVc6sVErdfTBPw355St6bHz91LUoDD -0S/GjUif8LYhVlZGXlwjYmVZOb2Z7+DAamzjwVrSsrxGCJ66Coy1rKapJuHVsfGA -W44p2ioIEZT3s6nQJkCt7te4ab1iWzaydZGAYyBao0K7kfK3vSp4AXsE5t5y6dpH -Yo0CAwEAAaOCAp0wggKZMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQD -AgGGMIGJBggrBgEFBQcBAQR9MHswMAYIKwYBBQUHMAGGJGh0dHA6Ly9jb21tZXJj -aWFsLm9jc3AuaWRlbnRydXN0LmNvbTBHBggrBgEFBQcwAoY7aHR0cDovL3ZhbGlk -YXRpb24uaWRlbnRydXN0LmNvbS9yb290cy9jb21tZXJjaWFscm9vdGNhMS5wN2Mw -HwYDVR0jBBgwFoAU7UQZwNPwBovupHu+QucmVMiONnYwggFEBgNVHSAEggE7MIIB -NzAIBgZngQwBBAIwDQYLYIZIAYb5LwAGDQMwggEaBgtghkgBhvkvAAYNATCCAQkw -SgYIKwYBBQUHAgEWPmh0dHBzOi8vc2VjdXJlLmlkZW50cnVzdC5jb20vY2VydGlm -aWNhdGVzL3BvbGljeS90cy9pbmRleC5odG1sMIG6BggrBgEFBQcCAjCBrQyBqlRo -aXMgVHJ1c3RJRCBDZXJ0aWZpY2F0ZSBoYXMgYmVlbiBpc3N1ZWQgaW4gYWNjb3Jk -YW5jZSB3aXRoIElkZW5UcnVzdCdzIFRydXN0SUQgQ2VydGlmaWNhdGUgUG9saWN5 -IGZvdW5kIGF0IGh0dHBzOi8vc2VjdXJlLmlkZW50cnVzdC5jb20vY2VydGlmaWNh -dGVzL3BvbGljeS90cy9pbmRleC5odG1sMEoGA1UdHwRDMEEwP6A9oDuGOWh0dHA6 -Ly92YWxpZGF0aW9uLmlkZW50cnVzdC5jb20vY3JsL2NvbW1lcmNpYWxyb290Y2Ex -LmNybDAdBgNVHQ4EFgQUyjLwNnzHKtqRtXyHihG9uCJsvwkwEwYDVR0lBAwwCgYI -KwYBBQUHAwgwDQYJKoZIhvcNAQELBQADggIBACtnO4f6QB6v2yDFeld3Pa1H7Bmb -y2tSwzQ/5dB5WXmTZHlV433s7BDkpwGzK4fGBuTcx814uPUWWSwcL+f8bpfozBv6 -p855j/AlKul1EiPMKkMlLtwdiK6sWT2x8qaTm4fMGbcpgUbQnxOo4BnzVUmgs3ep -m9/qXf9GaWXRz4maSWl4z3apD3X/5oMrriGWIiW6ivCq8bmOBUdm0I9quhW9Snk2 -JAaqkVCjs06rqE3rRblyNdrSypGzo5eBT498aCfcvDPJX2/q2PMkLvkKoXtVJ1g4 -sEwDQxm2sg8QMEd2GKo+X7TqOeF8An7KOPDq9v0xtSsF4+ufFrl43vl6v4uMey68 -wOHv6VmaGpCtWk1e6lSq9jLQqRBBg8CMwpw6niVyvkrdh4Tvu+5HLrareZBp98PJ -2cQzrHk1SiPcyDxSlhbXRks/TgKXTicUm3pZsKZcTvCXHcqulN1eoQ3z9azMIuR7 -RtdECz2RJUI6Xg//6ZdMMgaDnktMALgAyo9GgGGVimx4E2/aM5r0cm+RCrk956BT -qahEdiGLWjgjq8dAJe4XfLtp6EmGvsw650uBbDA0Mg9nlSCFdGTMFBKlw65o2oJa -enMysAvE/VC39ZIEOBfk2IOj2GTYlOgHi0iIBIZf7SqdjhTAtoW7U9TTZj8SHRen -/EEe1WEcfTawcOhJ ------END CERTIFICATE----- diff --git a/tests/data/timestamping/leaf.crt b/tests/data/timestamping/leaf.crt deleted file mode 100644 index 539ae6e..0000000 --- a/tests/data/timestamping/leaf.crt +++ /dev/null @@ -1,39 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIG4zCCBMugAwIBAgIQQAGSoSzKR+4rZmhPudMJdjANBgkqhkiG9w0BAQsFADBF -MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MSIwIAYDVQQDExlUcnVz -dElEIFRpbWVzdGFtcGluZyBDQSAzMB4XDTI0MTAxODE5NDg0MFoXDTI2MDExNzE5 -NDgzOVowRzELMAkGA1UEBhMCVVMxEjAQBgNVBAoTCUlkZW5UcnVzdDEkMCIGA1UE -AxMbVHJ1c3RJRCBUaW1lc3RhbXAgQXV0aG9yaXR5MIICIjANBgkqhkiG9w0BAQEF -AAOCAg8AMIICCgKCAgEA27BXl6MHeJwySCub5fd8rD4bWC2yOQE4hWDkw6YjMHMl -mKQVPIqD2+ezxhIK5GmwwNxDopo1ovsTMrF/KFsTExw0l4ZguJ+xs5W6qLrRVm7Z -TnkB6wMKaxR8+fdxtq9lnqRDnHtrGiGEQGOqVvUpAhnzioYa/5qWmqhRhdf7bRpI -1D71LC4UtWm1YG0PmJVSsE2ZqjB4rP7baj6hAlQxe0bvNwL/Bqy6D0QHNk2Xf2Vk -w7BLN7knh/SngAlzfumBg0cQff0C5DQgeyCQoM5/XBAyYZUQc//t2XVZPYiGUvOj -QNjLCbqOpRQf9JBaU/gh9Mf6Mbs6xh5qn4gKHpxOUMy+QfYAU+RiLhV4X0/brdqc -UOqRHte9gcv8yU4mqyMzbmHn5y09SkuhPOIhYE5Fd72XHeR8RnC9CI9V5NjVW370 -iNZuO2R2hw3dwtlWFxJPM+jortoxO3BPoxmppp/EdEH60p5Sk12eeyx86zieLb1E -JS5gEvu+jYWwKKjCf3visWC4OJNzbknGr3WRVOI6UvjyKrck3hpCuOjLXST+pEiB -XJzbiHCR5UJn8mXJnPAvKabgCGk7/trqrqB3Rkd2l5rce7maoyn265r/3IRLZisf -QdBsfzQv6RAP/zBFQkzcE2Yl12M25k2B6kFZMLsTT60jBTt1W6utXM2T7DbOwO8C -AwEAAaOCAcswggHHMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBYGA1Ud -JQEB/wQMMAoGCCsGAQUFBwMIMIGFBggrBgEFBQcBAQR5MHcwMAYIKwYBBQUHMAGG -JGh0dHA6Ly9jb21tZXJjaWFsLm9jc3AuaWRlbnRydXN0LmNvbTBDBggrBgEFBQcw -AoY3aHR0cDovL3ZhbGlkYXRpb24uaWRlbnRydXN0LmNvbS9jZXJ0cy90aW1lc3Rh -bXBpbmczLnA3YzAfBgNVHSMEGDAWgBTKMvA2fMcq2pG1fIeKEb24Imy/CTB/BgNV -HSAEeDB2MAgGBmeBDAEEAjANBgtghkgBhvkvAAYNATBbBgtghkgBhvkvAAYNAzBM -MEoGCCsGAQUFBwIBFj5odHRwczovL3NlY3VyZS5pZGVudHJ1c3QuY29tL2NlcnRp -ZmljYXRlcy9wb2xpY3kvdHMvaW5kZXguaHRtbDBGBgNVHR8EPzA9MDugOaA3hjVo -dHRwOi8vdmFsaWRhdGlvbi5pZGVudHJ1c3QuY29tL2NybC90aW1lc3RhbXBpbmcz -LmNybDAdBgNVHQ4EFgQUIYBfL7FSq7kLBlpOTWDaRGHtFgEwDQYJKoZIhvcNAQEL -BQADggIBAFIczKS3kFMFeX2WIC1uUj4Nvt+W2/kPNhuKukF2pOC+VcxrbbsTugTx -oO+X8J5JVX5sNP74p7YrkSY+dVPFQ+8rfJJeshSvJEQnt9DgsXPTRcU982OPmJCQ -QZY2Ux7xYfoEuWytuigyhMMoh+g5IzipQ56UCDY/sHY40SdLpSXg69RMy6y/L2Zm -9f4YPFrXPy7q3hGTemQHo+jmshg/hU/zIEjjfWx7uG223r0M5Ez9ks4y/EmtdT5l -KkF9RGALpKEBWIQaL+yi8X8NHRM1Qfs91GvEZe5wPri/5R9YvhiHWjizqxvDlqVW -ka6Mu13zbovM2vMppHJWYlvBTZ4z8vZBdNzN7fiTAdcd3lAl0A3wmpBIqrvFChhY -DX0Su1/3kA3X2PAJrbaZ7RNQ+Zjuz31T8QK0d1PyzHOZ/jK76f7Nb5Ic8fbALKzs -S+Yg3O4fmLqC0kERtL96xA/y+Y9oAJYMOOvrpBvpe1TdGZ5s3nRpLhWT4NEddkGg -AWodbBA3W0x5iKRWXAo+sATHTNXB6RGTIuF1/PC3R4fjbTMvpHXyW+bP+k8m+uIu -XLDMJ+XW5TZ01g6UGfg62ti4ohS0PyEHbnuEQDQogcqtqF0d8oihfcbDXOm/YgId -vhdRXJC/UZ0Q6Q6jtuj5uV76fvNU4j4FYKnXZqutzlCxeKn1odHN ------END CERTIFICATE----- diff --git a/tests/data/timestamping/root.crt b/tests/data/timestamping/root.crt deleted file mode 100644 index de6f07d..0000000 --- a/tests/data/timestamping/root.crt +++ /dev/null @@ -1,31 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFYDCCA0igAwIBAgIQCgFCgAAAAUUjyES1AAAAAjANBgkqhkiG9w0BAQsFADBK -MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScwJQYDVQQDEx5JZGVu -VHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwHhcNMTQwMTE2MTgxMjIzWhcNMzQw -MTE2MTgxMjIzWjBKMQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScw -JQYDVQQDEx5JZGVuVHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwggIiMA0GCSqG -SIb3DQEBAQUAA4ICDwAwggIKAoICAQCnUBneP5k91DNG8W9RYYKyqU+PZ4ldhNlT -3Qwo2dfw/66VQ3KZ+bVdfIrBQuExUHTRgQ18zZshq0PirK1ehm7zCYofWjK9ouuU -+ehcCuz/mNKvcbO0U59Oh++SvL3sTzIwiEsXXlfEU8L2ApeN2WIrvyQfYo3fw7gp -S0l4PJNgiCL8mdo2yMKi1CxUAGc1bnO/AljwpN3lsKImesrgNqUZFvX9t++uP0D1 -bVoE/c40yiTcdCMbXTMTEl3EASX2MN0CXZ/g1Ue9tOsbobtJSdifWwLziuQkkORi -T0/Br4sOdBeo0XKIanoBScy0RnnGF7HamB4HWfp1IYVl3ZBWzvurpWCdxJ35UrCL -vYf5jysjCiN2O/cz4ckA82n5S6LgTrx+kzmEB/dEcH7+B1rlsazRGMzyNeVJSQjK -Vsk9+w8YfYs7wRPCTY/JTw436R+hDmrfYi7LNQZReSzIJTj0+kuniVyc0uMNOYZK -dHzVWYfCP04MXFL0PfdSgvHqo6z9STQaKPNBiDoT7uje/5kdX7rL6B7yuVBgwDHT -c+XvvqDtMwt0viAgxGds8AgDelWAf0ZOlqf0Hj7h9tgJ4TNkK2PXMl6f+cB7D3hv -l7yTmvmcEpB4eoCHFddydJxVdHixuuFucAS6T6C6aMN7/zHwcz09lCqxC0EOoP5N -iGVreTO01wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB -/zAdBgNVHQ4EFgQU7UQZwNPwBovupHu+QucmVMiONnYwDQYJKoZIhvcNAQELBQAD -ggIBAA2ukDL2pkt8RHYZYR4nKM1eVO8lvOMIkPkp165oCOGUAFjvLi5+U1KMtlwH -6oi6mYtQlNeCgN9hCQCTrQ0U5s7B8jeUeLBfnLOic7iPBZM4zY0+sLj7wM+x8uwt -LRvM7Kqas6pgghstO8OEPVeKlh6cdbjTMM1gCIOQ045U8U1mwF10A0Cj7oV+wh93 -nAbowacYXVKV7cndJZ5t+qntozo00Fl72u1Q8zW/7esUTTHHYPTa8Yec4kjixsU3 -+wYQ+nVZZjFHKdp2mhzpgq7vmrlR94gjmmmVYjzlVYA211QC//G5Xc7UI2/YRYRK -W2XviQzdFKcgyxilJbQN+QHwotL0AMh0jqEqSI5l2xPE4iUXfeu+h1sXIFRRk0pT -AwvsXcoz7WL9RccvW9xYoIA55vrX/hMUpu09lEpCdNTDd1lzzY9GvlU47/rokTLq -l1gEIt44w8y8bckzOmoKaT+gyOpyj4xjhiO9bTyWnpXgSUyqorkqG5w2gXjtw+hG -4iZZRHUe2XWJUc0QhJ1hYMtd+ZciTY6Y5uN/9lu7rs3KSoFrXgvzUeF0K+l+J6fZ -mUlO+KWA2yUPHGNiiskzZ2s8EIPGrd6ozRaOjfAHN3Gf8qv8QfXBi+wAN10J5U6A -7/qxXDgGpRtK4dw4LTzcqx+QGtVKnO7RcGzM7vRX+Bi6hG6H ------END CERTIFICATE----- diff --git a/tests/data/timestamping/self_signed.tsr b/tests/data/timestamping/self_signed.tsr new file mode 100644 index 0000000000000000000000000000000000000000..e7ffd8344a4476af45816719931c4864d7bc8b1f GIT binary patch literal 1600 zcmbW1X;71A5XbYrZxRwn2*^1o5{VWGf=>wHQ0@k#BBIElI4BsRBnSwhq*$yZ#T3EH zA&RzxNjNluNI)4u@F=K+dlV?(2q;mmQVEL6q0-oKDt_n}KR(a?XJ_}n`&%HyIs*e3 zLIE9uw*Mh^LPR1bE|%;c9gz%#*i~3ORpcXjON2CFi4eO?{iO{<2#o}YSbVt12Vx^= z7{Z`{pa84&Nr~5i&=SF~Xh8-L`~pJ|7NUafOZ5l_g?3+vzkm|QNoO`zBy#bsET^HP zCf%_klA}XgFp0*#HA>O;~sl){|F$Y?yo1Qcd&tjKR+9F|{#GrQ_ z$T3Id+KX1CX%BaqG07I#G60=MquJTfY#l>kJnD_($!9t1YQrQ#!~+Ns7a)Y!NDvr; zVO><3@7v{&#oba;k*TbTv$O7FiJw}4wA3xx(@cOd770RY5_I`|z73CaAa+0JAe(2y zii@Ec1H+Fogh296V6(}AJ3Poc*a=DO1hCSGNOJ@XIsoP>1yVQX*VuPr?g*P^gdWWIoc+jM1b(#Fxfo(?J zITHcv9%XYn+{;ZBxCb(Rz2Zxkd$}PG?lCQ-0eX~Ig~wk#G4}|+ zUeiN4d^^)w>YVtn%9JQoQl%|AJigF(x9jQyY5T(F@6>K*s(LE5OXX#uIC9Vjx;I_r zMZlfkOY*Dmb5#s#Ql@K5B8wa=E~OS1dRevx`ri*#bWdHGi8=O8qp2bxkKf2-;Lmh& ze;kwTE^Sq0=)wquf{MHW6KJU?V@g7!5m+Pj-!Yl4fuKb+0cc`zR#*sw(Liv(^rJ`{ zZHU&d&fXQwis*0TraY|E3+~d~oSHtk)M@oHsGWvT&Q0m=CDuW~q2(<$devhduV`63 z-8pUpN@qhe;uKf7s_$O-7)yM|!In2WcH(()wYkF4PaL1_+w#I_F1M_3yHq!FwMIbk z9=n(|izFgO4NKm5_3~r0l$~qGeh^oh()$m|b$&WXwch3Nl2L=j z-p;+#JifW@G1hH(ADe&orOaxT zY{eXdkz!Z^XswRwi%UwPAFwzN^=hCWLj>01?$p1K+P*8DzfO{n+@q<1|Gj2sd$mKRNPbYjBCY#CRfY zxGpFuecera&e_^%!iN?M!5?7_TNZOWwe*#zCqrhTYh>oc5aak`$xApeJB^@dG}CXVB+olT361ye4qDbx|Q*~qcivUKVj)=Q2+n{ literal 0 HcmV?d00001 diff --git a/tests/data/timestamping/timestamp_token_http-timestamp-identrust-com b/tests/data/timestamping/timestamp_token_http-timestamp-identrust-com new file mode 100644 index 0000000000000000000000000000000000000000..1def8d13da9563906351e1de95e369cdcac3c485 GIT binary patch literal 4767 zcmb`Kby$?!_Q#oEXa@-iMTU-%;T<}pkuE_v5>mq;qeI9HND2%oC?N`nNC?s(ijopa zsDxr5?Lj~Z=>}2mj6d~q&h_{BJolbIzx&;>*1PvwpS1uI%MBQvc$8k$1`~(|oK0f6 z1dv!_z#s@34lu#!9HaC=SO^phqNW1E3SbQ1XVNi)pxI%m(6FmOSP~cnf-z7gMr}z_ z!I&Vx@x3b{)KpOQ&*$=~vOIHEmG2PG&~;)I{|w=N@rIM_a}=VJre&E?{PL(6xx}VGwP(Z&?D+FP+N?mja|@0ckWE z4ai8#D%gPOs8+mkfH|Fsm~7I%Tz~`{2T0J)YzP&5S`bP{xG)_SuFD+!1w0{x2t*`iKM%2EmU6nrOxT?p@Ud2ji85cwRqyYl ztBb$Q#=DwmW3kD%`eYkjuhf3{OjZp2;S3YAu11Y@DvJqc6uX~;=}_5VhS^Ng;l75j z&Yfo_!X*OI9!`^!5-J9UIY-C%98Hd$Z%E~t2-5LtpiQux=(c9X5f*aPYRh54cbB|N0!fxmm@1g3m=CB zj@@;vQAj3u=(peTNjfp6WWKp9UM={JTeD`W^OWG%JiRcxjG-8>qz{^>m$A<>7A3{< zM`>}&Uk7K(v(M}3`lhB22iB(NimiW2-yM!HbVc%N^x#8eB&V6-yTCC`qhVH8p+1b< zN8_*=4QrsD)wE22TG_|8q|B~6c~y^aNlezbGD75NH<)2B?62 z6a}X!CJsh20{yjMg9;V5Y!(y4*)+j0E!ZHBNDxoBqcGP z?(SG`49*FU^289lP&jhD5GnD3B7Xt2ey#vV$$wwr>x9R-I1zE49)F!IiNSgk36jJ; zK_!jylE(nN6#E3AoWOxlwhpO9xu7TFgGmp9qGR_r)bQh*=ztK)1O(I%@C9gLu+w3T zAhMrKdvg*9u-z*mfDvHu!!$7HfGWi_0@)h^7UScM{e@jPR~*KPh$TpRdE#*xe@U{T zI1d-BAIgpBjt6uo2Jk}F02M$vTQOTcO7^$DdgK4*DaIQQ@KE;I2R#5~;|q0=ylY-k z%K)=6F?Jl(cA7f?+8glXoJ9_J5S96%uDt3*3u^!16oiC_xhhlRpD9CIaJhrAbsD~T zqm7nbXWi>r>!3^73#p5l`j!{)GG8|fs*p*-sy-HNS_{aaEB5!;?g(m&RQHVDCA>b? zrm})bNJ~uANRhKRyyUbEt;LtsiAqJZh(=E-3Mdq5WnQI~Q>*Y*cqw9dTWIvt8>7zB zCP`PvODTTWRwzf-RQ7_s?4_cM-0Cm8zfq+U zD~&hX%w?Hz zbY49crZexnssl}x6zi0A)3&O?i*xr%#=bf@u3iwgV_X3rh@35Q40@I(uqvZC0Rm^b z$eAZRDuR{@EbhD0CsCA<$3Kv9cB}nTL1n5nm!Yeu>~pSowR-ghXSwO&t4vBkoO;`W zG@zpT*j5IlSH&sj&aNWH%KKv#9WeEYn>{b!(@|>b@6(!g%NV$_8jRMtWh*sk@=3_J z_@x{*dex|ErtZ?xbrskggE~r+DE9f8;S}m2@>Pv)%kz;(Zp&iuGj&{~Mzr4L*zWlj zy3sSUabpb%D>RJe5ioA0Z^H8|VK-SbHb#UN~QHS3d>s~g1qK!MPY@4Tl6b)7GNCmr}E zx%RmZuuwASH`37pFf!7}De|wp14RL07k{LFGS`BE!2g14x<7URz){M-%k|#mkRj{= zB}HSL(Eq`Y`MkhJ%}L=CPvhl$$@tc$xP*X`K+0qx|NjvVLjK6TmvJES z0Np=Ixv;t)UW36vSQ%qj@d-B^0r8_u)gt5jJ0=lsP6UKA7VCk)5ePn57X;1&;e^3> zdb>DzV6ceuIHDWkXLb?!^%iE|K#Oqo^zm>(I1v&5Bccr`e~}le1!w?j*(%w}QHsC2 zHZqWZacO_W^e@X1R?1OAe=DZ`+HZVgH}bS{^tSrpJ41-JfIrTwlGrExJ79g}kmTf!N9t79!R>+FGOW_x)i(M>Us*R`Cb?r$wQ^($5;4dc#D^4=Ytu!eqH2!p~)?0S) zRYy`~ZfclmLXext@I&cm4xTgpD}tyk`kX*Zk&_A)Mi4Eos(Wx|K-ZVESk>YFjNhx(ds3Zw#D%rz#r&EojVBwjDMpQLa9SJ(2PS;}b zeypB@(&?+-o=NnzDwEk3MTOeFd~r4k0wqt>h2}Ye>Ty&0QH@9F`WhmV z=+ec;a=WG6ZIEOWK#;0nX#HYVy>);*PI6J#gsX->A`Wa{s!w#|DslCW9H#hX8JXT$V%G=o68rj;{$h-iBbF7SIXU0FRG` z0FwXS3X0~3A|zC9WIg6*V85_#FsVoEYd7hQr%MW55si=B@2?h{D808>Avd&85}3*% zCNv5)WEfpehD*d;C}yB(F6#gph(Kcs)k;s~} zT2I$Rtsc?BRv;rg=1dXDui5mMoidV;DePr|18}{PvvGOqB{zllV-Nvmv*BXGUiKtD z?e*bj?rDWXRQj~Cy$|#e^y>S>TZ*q~-cne%soN4>cx|I>Nj+dM_%z!xMycrWzW|E4 BXl4Ka literal 0 HcmV?d00001 diff --git a/tests/data/timestamping/timestamp_response.tsr b/tests/data/timestamping/valid_timestamp.tsr similarity index 100% rename from tests/data/timestamping/timestamp_response.tsr rename to tests/data/timestamping/valid_timestamp.tsr diff --git a/tests/enrichers/test_timestamping_enricher.py b/tests/enrichers/test_timestamping_enricher.py index 1db503c..850096a 100644 --- a/tests/enrichers/test_timestamping_enricher.py +++ b/tests/enrichers/test_timestamping_enricher.py @@ -1,22 +1,103 @@ +from pathlib import Path import pytest -from auto_archiver.modules.timestamping_enricher.timestamping_enricher import TimestampingEnricher + from rfc3161_client import ( TimeStampResponse, decode_timestamp_response, ) +import requests -from cryptography import x509 +from auto_archiver.modules.timestamping_enricher.timestamping_enricher import TimestampingEnricher +from auto_archiver.core import Metadata, Media @pytest.fixture def timestamp_response() -> TimeStampResponse: - with open("tests/data/timestamping/timestamp_response.tsr", "rb") as f: + with open("tests/data/timestamping/valid_timestamp.tsr", "rb") as f: return decode_timestamp_response(f.read()) @pytest.fixture def wrong_order_timestamp_response() -> TimeStampResponse: with open("tests/data/timestamping/rfc3161-client-issue-104.tsr", "rb") as f: return decode_timestamp_response(f.read()) + +@pytest.fixture +def selfsigned_response() -> TimeStampResponse: + with open("tests/data/timestamping/self_signed.tsr", "rb") as f: + return decode_timestamp_response(f.read()) +@pytest.fixture +def filehash(): + return "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef" + +def test_full_enriching(setup_module, sample_media, mocker, timestamp_response, filehash): + mock_post = mocker.patch("requests.sessions.Session.post") + mock_post.return_value.status_code = 200 + mock_decode_timestamp_response = mocker.patch("auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response") + mock_decode_timestamp_response.return_value = timestamp_response + + tsp: TimestampingEnricher = setup_module("timestamping_enricher", {"tsa_urls": ["http://timestamp.identrust.com"]}) + metadata = Metadata().set_url("https://example.com") + sample_media.set('hash', filehash) + metadata.add_media(sample_media) + tsp.enrich(metadata) + + assert metadata.get('timestamped') == True + assert len(metadata.media) == 2 # the original 'sample_media' and the new 'timestamp_media' + + timestamp_media = metadata.media[1] + assert timestamp_media.filename == f"{tsp.tmp_dir}/hashes.txt" + assert Path(timestamp_media.filename).read_text() == filehash + + # we only have one authority file because we only used one TSA + assert len(timestamp_media.get('timestamp_authority_files')) == 1 + timestamp_authority_file = timestamp_media.get('timestamp_authority_files')[0] + assert Path(timestamp_authority_file.filename).read_bytes() == timestamp_response.time_stamp_token() + + cert_chain = timestamp_authority_file.get('cert_chain') + assert len(cert_chain) == 3 + assert cert_chain[0].filename == f"{tsp.tmp_dir}/1 – 85078758028491331763.crt" + assert cert_chain[1].filename == f"{tsp.tmp_dir}/2 – 85078371663472981624.crt" + assert cert_chain[2].filename == f"{tsp.tmp_dir}/3 – 13298821034946342390.crt" + +def test_full_enriching_multiple_tsa(setup_module, sample_media, mocker, timestamp_response, filehash): + mock_post = mocker.patch("requests.sessions.Session.post") + mock_post.return_value.status_code = 200 + + mock_decode_timestamp_response = mocker.patch("auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response") + mock_decode_timestamp_response.return_value = timestamp_response + + tsp: TimestampingEnricher = setup_module("timestamping_enricher", {"tsa_urls": ["http://example.com/timestamp1", "http://example.com/timestamp2"]}) + metadata = Metadata().set_url("https://example.com") + sample_media.set('hash', filehash) + metadata.add_media(sample_media) + tsp.enrich(metadata) + + assert metadata.get('timestamped') == True + assert len(metadata.media) == 2 # the original 'sample_media' and the new 'timestamp_media' + + timestamp_media = metadata.media[1] + assert len(timestamp_media.get('timestamp_authority_files')) == 2 + for timestamp_token_media in timestamp_media.get('timestamp_authority_files'): + assert Path(timestamp_token_media.filename).read_bytes() == timestamp_response.time_stamp_token() + assert len(timestamp_token_media.get('cert_chain')) == 3 + + + + + + + + + + +def test_enriching(setup_module, sample_media): + tsp: TimestampingEnricher = setup_module("timestamping_enricher") + + # test the enrich method + metadata = Metadata().set_url("https://example.com") + sample_media.set('hash', "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef") + metadata.add_media(sample_media) + tsp.enrich(metadata) @pytest.mark.download def test_fails_for_digicert(setup_module): @@ -57,8 +138,8 @@ def test_verify_save(setup_module, timestamp_response): cert_chain = tsp.save_certificate(timestamp_response, verified_root_cert) assert len(cert_chain) == 3 - assert cert_chain[0].filename == f"{tsp.tmp_dir}/1 – 85078371663472981624.crt" - assert cert_chain[1].filename == f"{tsp.tmp_dir}/2 – 85078758028491331763.crt" + assert cert_chain[0].filename == f"{tsp.tmp_dir}/1 – 85078758028491331763.crt" + assert cert_chain[1].filename == f"{tsp.tmp_dir}/2 – 85078371663472981624.crt" assert cert_chain[2].filename == f"{tsp.tmp_dir}/3 – 13298821034946342390.crt" @@ -70,7 +151,25 @@ def test_order_crt_correctly(setup_module, wrong_order_timestamp_response): ordered_certs = tsp.tst_certs(wrong_order_timestamp_response) assert len(ordered_certs) == 2 - assert ordered_certs[0].subject.rfc4514_string() == "CN=TrustID Timestamping CA 3,O=IdenTrust,C=US" - assert ordered_certs[1].subject.rfc4514_string() == "CN=TrustID Timestamp Authority,O=IdenTrust,C=US" + assert ordered_certs[0].subject.rfc4514_string() == "CN=TrustID Timestamp Authority,O=IdenTrust,C=US" + assert ordered_certs[1].subject.rfc4514_string() == "CN=TrustID Timestamping CA 3,O=IdenTrust,C=US" +def test_invalid_tsa_404(setup_module, mocker): + tsp = setup_module("timestamping_enricher") + post_mock = mocker.patch("requests.sessions.Session.post") + post_mock.side_effect = Exception("error") + with pytest.raises(Exception, match="error"): + tsp.sign_data("http://bellingcat.com/", b"my-message") + +@pytest.mark.download +def test_invalid_tsa_invalid_response(setup_module, mocker): + tsp = setup_module("timestamping_enricher") + + with pytest.raises(requests.exceptions.HTTPError, match="404 Client Error"): + tsp.sign_data("http://bellingcat.com/page-not-found/", b"my-message") + +def test_fail_on_selfsigned_cert(setup_module, selfsigned_response): + tsp = setup_module("timestamping_enricher") + root_cert = tsp.verify_signed(selfsigned_response, b"my-message") + assert root_cert is None \ No newline at end of file diff --git a/tests/storages/test_local_storage.py b/tests/storages/test_local_storage.py index 85f97c6..c2bf1f0 100644 --- a/tests/storages/test_local_storage.py +++ b/tests/storages/test_local_storage.py @@ -18,22 +18,11 @@ def local_storage(setup_module) -> LocalStorage: } return setup_module("local_storage", configs) - -@pytest.fixture -def sample_media(tmp_path) -> Media: - """Fixture creating a Media object with temporary source file""" - src_file = tmp_path / "source.txt" - src_file.write_text("test content") - return Media(key="subdir/test.txt", filename=str(src_file)) - - def test_get_cdn_url_relative(local_storage): media = Media(key="test.txt", filename="dummy.txt") expected = os.path.join(local_storage.save_to, media.key) assert local_storage.get_cdn_url(media) == expected - - def test_get_cdn_url_absolute(local_storage): media = Media(key="test.txt", filename="dummy.txt") local_storage.save_absolute = True From 89ee6f19b69aefaf46bfdbe172ae8717a2c82287 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Tue, 11 Mar 2025 16:12:13 +0000 Subject: [PATCH 12/20] List out all valid TSAs + option for users to allow self signed if they want --- .../timestamping_enricher/__manifest__.py | 19 ++++--- .../timestamping_enricher.py | 5 +- tests/enrichers/test_timestamping_enricher.py | 53 ++++++++++++------- 3 files changed, 51 insertions(+), 26 deletions(-) diff --git a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py index 2c81dbb..c16502d 100644 --- a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py +++ b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py @@ -18,18 +18,17 @@ # See https://github.com/trailofbits/rfc3161-client/issues/46 for a list of valid TSAs # Full list of TSAs: https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710 "http://timestamp.identrust.com", - "http://timestamp.ssl.trustwave.com", #timeouts + "http://timestamp.ssl.trustwave.com", "http://zeitstempel.dfn.de", "http://ts.ssl.com", - "http://tsa.izenpe.com", + # "http://tsa.izenpe.com", # self-signed "http://tsa.lex-persona.com/tsa", - "http://ca.signfiles.com/TSAServer.aspx", - "http://aloahacoin.chain-provider.com/tsa.aspx", - "http://tsa.sinpe.fi.cr/tsaHttp/", - "http://tsa.cra.ge/signserver/tsa?workerName=qtsa", + # "http://ca.signfiles.com/TSAServer.aspx", # self-signed + # "http://tsa.sinpe.fi.cr/tsaHttp/", # self-signed + # "http://tsa.cra.ge/signserver/tsa?workerName=qtsa", # self-signed "http://tss.cnbs.gob.hn/TSS/HttpTspServer", "http://dss.nowina.lu/pki-factory/tsa/good-tsa", - "https://freetsa.org/tsr", + # "https://freetsa.org/tsr", # self-signed ], "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.", }, @@ -37,6 +36,12 @@ "default": None, "help": "Path to a file containing trusted Certificate Authorities (CAs) in PEM format. If empty, the default system authorities are used.", "type": "str", + }, + "allow_selfsigned": { + "default": False, + "help": "Whether or not to allow and save self-signed Timestamping certificates. This allows for a greater range of timestamping servers to be used, \ +but they are not trusted authorities", + "type": "bool" } }, "description": """ diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index 6f9aa56..1f376da 100644 --- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -82,7 +82,10 @@ class TimestampingEnricher(Enricher): # fail if there's any issue with the certificates, uses certifi list of trusted CAs or the user-defined `cert_authorities` root_cert = self.verify_signed(signed, message) if not root_cert: - raise ValueError(f"No valid root certificate found for {tsa_url=}. Are you sure it's a trusted TSA? Or define an alternative trusted root with `cert_authorities`. (tried: {self.cert_authorities or certifi.where()})") + if self.allow_selfsigned: + logger.warning(f"Allowing self-signed certificat from TSA {tsa_url=}") + else: + raise ValueError(f"No valid root certificate found for {tsa_url=}. Are you sure it's a trusted TSA? Or define an alternative trusted root with `cert_authorities`. (tried: {self.cert_authorities or certifi.where()})") # save the timestamping certificate cert_chain = self.save_certificate(signed, root_cert) diff --git a/tests/enrichers/test_timestamping_enricher.py b/tests/enrichers/test_timestamping_enricher.py index 850096a..2f78551 100644 --- a/tests/enrichers/test_timestamping_enricher.py +++ b/tests/enrichers/test_timestamping_enricher.py @@ -29,6 +29,41 @@ def selfsigned_response() -> TimeStampResponse: def filehash(): return "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef" +@pytest.mark.download +def test_enriching(setup_module, sample_media): + tsp: TimestampingEnricher = setup_module("timestamping_enricher") + + # tests the current TSAs set as default in the __manifest__ to make sure they are all still working + + # test the enrich method + metadata = Metadata().set_url("https://example.com") + sample_media.set('hash', "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef") + metadata.add_media(sample_media) + tsp.enrich(metadata) + + +def test_full_enriching_selfsigned(setup_module, sample_media, mocker, selfsigned_response, filehash): + mock_post = mocker.patch("requests.sessions.Session.post") + mock_post.return_value.status_code = 200 + mock_decode_timestamp_response = mocker.patch("auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response") + mock_decode_timestamp_response.return_value = selfsigned_response + + tsp: TimestampingEnricher = setup_module("timestamping_enricher", {"tsa_urls": ["http://timestamp.identrust.com"]}) + metadata = Metadata().set_url("https://example.com") + sample_media.set('hash', filehash) + metadata.add_media(sample_media) + tsp.enrich(metadata) + + assert len(metadata.media) == 1 # doesn't allow self-signed + + # set self-signed on tsp + tsp.allow_selfsigned = True + + tsp.enrich(metadata) + + assert len(metadata.media) + + def test_full_enriching(setup_module, sample_media, mocker, timestamp_response, filehash): mock_post = mocker.patch("requests.sessions.Session.post") mock_post.return_value.status_code = 200 @@ -81,24 +116,6 @@ def test_full_enriching_multiple_tsa(setup_module, sample_media, mocker, timesta assert Path(timestamp_token_media.filename).read_bytes() == timestamp_response.time_stamp_token() assert len(timestamp_token_media.get('cert_chain')) == 3 - - - - - - - - - -def test_enriching(setup_module, sample_media): - tsp: TimestampingEnricher = setup_module("timestamping_enricher") - - # test the enrich method - metadata = Metadata().set_url("https://example.com") - sample_media.set('hash', "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef") - metadata.add_media(sample_media) - tsp.enrich(metadata) - @pytest.mark.download def test_fails_for_digicert(setup_module): """ From e811196711a7e1873eafba86ea70f658103febc1 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Mon, 24 Mar 2025 15:10:46 +0400 Subject: [PATCH 13/20] Ruff fixes --- tests/conftest.py | 2 + tests/enrichers/test_timestamping_enricher.py | 71 ++++++++++++------- tests/storages/test_local_storage.py | 1 + 3 files changed, 49 insertions(+), 25 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6d7b255..a9f9ff8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -139,6 +139,7 @@ def mock_binary_dependencies(mocker): mock_shutil_which.return_value = "/usr/bin/fake_binary" return mock_shutil_which + @pytest.fixture def sample_media(tmp_path) -> Media: """Fixture creating a Media object with temporary source file""" @@ -146,6 +147,7 @@ def sample_media(tmp_path) -> Media: src_file.write_text("test content") return Media(key="subdir/test.txt", filename=str(src_file)) + @pytest.fixture def sample_datetime(): return datetime(2023, 1, 1, 12, 0, tzinfo=timezone.utc) diff --git a/tests/enrichers/test_timestamping_enricher.py b/tests/enrichers/test_timestamping_enricher.py index 2f78551..706678d 100644 --- a/tests/enrichers/test_timestamping_enricher.py +++ b/tests/enrichers/test_timestamping_enricher.py @@ -8,27 +8,32 @@ from rfc3161_client import ( import requests from auto_archiver.modules.timestamping_enricher.timestamping_enricher import TimestampingEnricher -from auto_archiver.core import Metadata, Media +from auto_archiver.core import Metadata + @pytest.fixture def timestamp_response() -> TimeStampResponse: with open("tests/data/timestamping/valid_timestamp.tsr", "rb") as f: return decode_timestamp_response(f.read()) + @pytest.fixture def wrong_order_timestamp_response() -> TimeStampResponse: with open("tests/data/timestamping/rfc3161-client-issue-104.tsr", "rb") as f: return decode_timestamp_response(f.read()) - + + @pytest.fixture def selfsigned_response() -> TimeStampResponse: with open("tests/data/timestamping/self_signed.tsr", "rb") as f: return decode_timestamp_response(f.read()) + @pytest.fixture def filehash(): return "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef" + @pytest.mark.download def test_enriching(setup_module, sample_media): tsp: TimestampingEnricher = setup_module("timestamping_enricher") @@ -37,7 +42,7 @@ def test_enriching(setup_module, sample_media): # test the enrich method metadata = Metadata().set_url("https://example.com") - sample_media.set('hash', "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef") + sample_media.set("hash", "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef") metadata.add_media(sample_media) tsp.enrich(metadata) @@ -45,16 +50,18 @@ def test_enriching(setup_module, sample_media): def test_full_enriching_selfsigned(setup_module, sample_media, mocker, selfsigned_response, filehash): mock_post = mocker.patch("requests.sessions.Session.post") mock_post.return_value.status_code = 200 - mock_decode_timestamp_response = mocker.patch("auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response") + mock_decode_timestamp_response = mocker.patch( + "auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response" + ) mock_decode_timestamp_response.return_value = selfsigned_response tsp: TimestampingEnricher = setup_module("timestamping_enricher", {"tsa_urls": ["http://timestamp.identrust.com"]}) metadata = Metadata().set_url("https://example.com") - sample_media.set('hash', filehash) + sample_media.set("hash", filehash) metadata.add_media(sample_media) tsp.enrich(metadata) - assert len(metadata.media) == 1 # doesn't allow self-signed + assert len(metadata.media) == 1 # doesn't allow self-signed # set self-signed on tsp tsp.allow_selfsigned = True @@ -67,54 +74,62 @@ def test_full_enriching_selfsigned(setup_module, sample_media, mocker, selfsigne def test_full_enriching(setup_module, sample_media, mocker, timestamp_response, filehash): mock_post = mocker.patch("requests.sessions.Session.post") mock_post.return_value.status_code = 200 - mock_decode_timestamp_response = mocker.patch("auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response") + mock_decode_timestamp_response = mocker.patch( + "auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response" + ) mock_decode_timestamp_response.return_value = timestamp_response tsp: TimestampingEnricher = setup_module("timestamping_enricher", {"tsa_urls": ["http://timestamp.identrust.com"]}) metadata = Metadata().set_url("https://example.com") - sample_media.set('hash', filehash) + sample_media.set("hash", filehash) metadata.add_media(sample_media) tsp.enrich(metadata) - assert metadata.get('timestamped') == True - assert len(metadata.media) == 2 # the original 'sample_media' and the new 'timestamp_media' + assert metadata.get("timestamped") is True + assert len(metadata.media) == 2 # the original 'sample_media' and the new 'timestamp_media' timestamp_media = metadata.media[1] assert timestamp_media.filename == f"{tsp.tmp_dir}/hashes.txt" assert Path(timestamp_media.filename).read_text() == filehash # we only have one authority file because we only used one TSA - assert len(timestamp_media.get('timestamp_authority_files')) == 1 - timestamp_authority_file = timestamp_media.get('timestamp_authority_files')[0] + assert len(timestamp_media.get("timestamp_authority_files")) == 1 + timestamp_authority_file = timestamp_media.get("timestamp_authority_files")[0] assert Path(timestamp_authority_file.filename).read_bytes() == timestamp_response.time_stamp_token() - cert_chain = timestamp_authority_file.get('cert_chain') + cert_chain = timestamp_authority_file.get("cert_chain") assert len(cert_chain) == 3 assert cert_chain[0].filename == f"{tsp.tmp_dir}/1 – 85078758028491331763.crt" assert cert_chain[1].filename == f"{tsp.tmp_dir}/2 – 85078371663472981624.crt" assert cert_chain[2].filename == f"{tsp.tmp_dir}/3 – 13298821034946342390.crt" + def test_full_enriching_multiple_tsa(setup_module, sample_media, mocker, timestamp_response, filehash): mock_post = mocker.patch("requests.sessions.Session.post") mock_post.return_value.status_code = 200 - mock_decode_timestamp_response = mocker.patch("auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response") + mock_decode_timestamp_response = mocker.patch( + "auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response" + ) mock_decode_timestamp_response.return_value = timestamp_response - tsp: TimestampingEnricher = setup_module("timestamping_enricher", {"tsa_urls": ["http://example.com/timestamp1", "http://example.com/timestamp2"]}) + tsp: TimestampingEnricher = setup_module( + "timestamping_enricher", {"tsa_urls": ["http://example.com/timestamp1", "http://example.com/timestamp2"]} + ) metadata = Metadata().set_url("https://example.com") - sample_media.set('hash', filehash) + sample_media.set("hash", filehash) metadata.add_media(sample_media) tsp.enrich(metadata) - assert metadata.get('timestamped') == True - assert len(metadata.media) == 2 # the original 'sample_media' and the new 'timestamp_media' + assert metadata.get("timestamped") is True + assert len(metadata.media) == 2 # the original 'sample_media' and the new 'timestamp_media' timestamp_media = metadata.media[1] - assert len(timestamp_media.get('timestamp_authority_files')) == 2 - for timestamp_token_media in timestamp_media.get('timestamp_authority_files'): - assert Path(timestamp_token_media.filename).read_bytes() == timestamp_response.time_stamp_token() - assert len(timestamp_token_media.get('cert_chain')) == 3 + assert len(timestamp_media.get("timestamp_authority_files")) == 2 + for timestamp_token_media in timestamp_media.get("timestamp_authority_files"): + assert Path(timestamp_token_media.filename).read_bytes() == timestamp_response.time_stamp_token() + assert len(timestamp_token_media.get("cert_chain")) == 3 + @pytest.mark.download def test_fails_for_digicert(setup_module): @@ -130,6 +145,7 @@ def test_fails_for_digicert(setup_module): tsp.sign_data(tsa_url, data) assert "ASN.1 parse error: ParseError" in str(e.value) + @pytest.mark.download def test_download_tsr(setup_module): tsa_url = "http://timestamp.identrust.com" @@ -146,10 +162,13 @@ def test_download_tsr(setup_module): cert_chain = tsp.save_certificate(result, verified_root_cert) assert len(cert_chain) == 3 + def test_verify_save(setup_module, timestamp_response): tsp: TimestampingEnricher = setup_module("timestamping_enricher") - verified_root_cert = tsp.verify_signed(timestamp_response, b"4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef") + verified_root_cert = tsp.verify_signed( + timestamp_response, b"4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef" + ) assert verified_root_cert.subject.rfc4514_string() == "CN=IdenTrust Commercial Root CA 1,O=IdenTrust,C=US" cert_chain = tsp.save_certificate(timestamp_response, verified_root_cert) @@ -179,14 +198,16 @@ def test_invalid_tsa_404(setup_module, mocker): with pytest.raises(Exception, match="error"): tsp.sign_data("http://bellingcat.com/", b"my-message") + @pytest.mark.download def test_invalid_tsa_invalid_response(setup_module, mocker): tsp = setup_module("timestamping_enricher") with pytest.raises(requests.exceptions.HTTPError, match="404 Client Error"): tsp.sign_data("http://bellingcat.com/page-not-found/", b"my-message") - + + def test_fail_on_selfsigned_cert(setup_module, selfsigned_response): tsp = setup_module("timestamping_enricher") root_cert = tsp.verify_signed(selfsigned_response, b"my-message") - assert root_cert is None \ No newline at end of file + assert root_cert is None diff --git a/tests/storages/test_local_storage.py b/tests/storages/test_local_storage.py index 2afdf59..1230e3d 100644 --- a/tests/storages/test_local_storage.py +++ b/tests/storages/test_local_storage.py @@ -33,6 +33,7 @@ def test_too_long_save_path(setup_module): with pytest.raises(SetupError): setup_module("local_storage", {"save_to": "long" * 100}) + def test_get_cdn_url_relative(local_storage): local_storage.filename_generator = "random" media = Media(filename="dummy.txt") From 396ec03bae2c1dfb5b3dc31f239ef00665c414ef Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Mon, 24 Mar 2025 15:26:22 +0400 Subject: [PATCH 14/20] Tidy up unit tests further + make more non-download --- .../timestamping_enricher.py | 3 -- tests/conftest.py | 2 +- tests/data/timestamping/digicert.tsr | Bin 0 -> 6012 bytes tests/enrichers/test_timestamping_enricher.py | 26 ++++++++++-------- 4 files changed, 15 insertions(+), 16 deletions(-) create mode 100644 tests/data/timestamping/digicert.tsr diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index 756c1c9..93d3ae8 100644 --- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -78,7 +78,6 @@ class TimestampingEnricher(Enricher): try: message = bytes(data_to_sign, encoding='utf8') - print(tsa_url) logger.debug(f"Timestamping {url=} with {tsa_url=}") signed: TimeStampResponse = self.sign_data(tsa_url, message) @@ -118,8 +117,6 @@ class TimestampingEnricher(Enricher): f.write(timestamp_token) return tst_path - trust_roots = [] - with open(certifi.where(), "rb") as f: def verify_signed(self, timestamp_response: TimeStampResponse, message: bytes) -> x509.Certificate: """ Verify a Signed Timestamp Response is trusted by a known Certificate Authority. diff --git a/tests/conftest.py b/tests/conftest.py index a9f9ff8..44d8058 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -145,7 +145,7 @@ def sample_media(tmp_path) -> Media: """Fixture creating a Media object with temporary source file""" src_file = tmp_path / "source.txt" src_file.write_text("test content") - return Media(key="subdir/test.txt", filename=str(src_file)) + return Media(_key="subdir/test.txt", filename=str(src_file)) @pytest.fixture diff --git a/tests/data/timestamping/digicert.tsr b/tests/data/timestamping/digicert.tsr new file mode 100644 index 0000000000000000000000000000000000000000..1648bdb4086e7dcce70041df967f3ba59efd817f GIT binary patch literal 6012 zcmchbc|6oz`^U{<-*;KZzVjV>WSx;EvW-0?QnoC!kliFp_GB%LvPMEhp)8SIvWqBd zqDV@T`i;tc>wfOv?|z=w@AZ1-ukShMI_F&1e7~RbKGy}{*#iLx7(@c#*?q_;C8G7C zw`f7+U;>`~B(duR7zBaS1GHq6C!_U2ry)=q`EF`1;Wl6~Zf)XOHKqFIu$Rscb z^wThU*NYqknEhoQRBOktm?Yn)U{mcj#)GoQiGeZD8StEdl77~adg24>bm2Qo8_2KC;Pqvq#bMnwV z2Y&?uq9z9B0;dKj$sm?cGG?%)1)LpV-D^-W(;?l@xFb&cW29gP=UimqYQW*W0SL3) z{s0W+k40lnyTbGoVCELu3cz6{n1#N!oT3s8wfMG451D1veU{D$K{lO zA3r;GT6sk{TuxC=Uh%NvVLL$QcPAJD`kziv?C%AK0lYuAXW#Gou?c!_6BzOTPyp~? zfrGJuNJ$`gFg@`lgQ>uH;@f`YfzJPw&R?DZ)Z2S6+TOVLdN-Aiqb*j=BYUo>WtR$o zA($Rfv|)}0G{vvnf-ju+Hn4G)tdx%*$fF(zO4qZNKc+T}xTd?NL80TmgTbkFi>(#V zy-V_zSrKl3zN>Rv>1>HOWTN`h0i%*z&sPn72png8wvXoJ9 zQQq^1w##O=$&ZXI-uLB9!tUC1heY+%5007*h}l^`N{kNNT7`2MhnoeChD2S+3XWY^ z`|^2g;5Oq${fe*(Wn;OHWcz|+m03=wCNx>@7nE#G*N5_2ztLw;4mxZTsE1lhBol|I z85EiGVJ4wXjITr3Ka#q0VxL|qLB3ko&ri{2!-h#h-Xlk7CChBmyzbcwl-57u7ka@Y zXUM7O#NPvh#D)4jIAZ9I#*POv?zC5?8|SX5pdI7l#M{r%+`3{0loXehE;C>@G zUqE;jf8B*=`jq#U`{HV+V;=nw>3qbb2E|&dd$WfHGB9n8{|%WiE{~L=Q_^8DBcz{p z@a+Q?Eg8mJUEQL7_Q`_HoUtO;MFL**>2bXnDJe6Ub42PkH+Z3FbQ?V?+O?EKZjT;U zT2hi@>5W(BSJYWHH6&TKfByENH3i?j+gy_pr{07n_wI^XzVTu+5l!#O2p2il%a%RR zbB%EK9ux!65x*uLdJyP46bb>8M*uW?ZRWi;7$5=I z_FBSwEgFD|j6#wOL`q5yp#Wfe{V?bufE}P9BRdmG10tGJA3Ol~_ACUU8~|GcOFi3! zxA8?;v^XwD7MtJ!OU~=`0eJQd=%8#+mV9d|NC2G}Yl>Tui{m?!%X~P3Jz%#t%m=j~ zo+Ks&W5Tg$gR>Znud1x9i@%q=j4M%-E<{brxcGR>?hE0U4D}R#iSVZe4@wZBEkm3R zM5gSA0s=uJF94AdfY5#q0E*`LwUdtv+V{7e0GmIoz|ltg{y6VBg8t3$KXnOnP_V?q zh$vVPiTW*GK3ul_oJtgv-UK4`6XWF7;}ks4#7v$JO$A3D6LBmL9E?xRJnyp8CQx7{ zF)b$EdS{Ky&7smV(nou(OkLp`DieQNlIDiFPC*(jtE1}cBkAHIV{Zv7wP7-L@?K_{ zySy=d%Z}Kp*4`7tJz=aKca|D2mY3PVBn=Zbz|Jw>Z_;Q{VAi@*V=82YMpW^E=T#f< znkW|VNArd_3Dgp$Rlb6AqBN|5MAd|Y8vk4924x& z#gxAIbfsSZ0`vZJ+yO^lb250m+m;ZtkI~b(SZ>b#Rw-3yK5T$7sW+<7zkrOmSx_f_cUpIU*d$MvDuH{=SK-Ao(;s+k#C9X1*M zflL`+$Nde{w5#YEqvO^fswvnn_mT6WnI(Xq(p3!x0LRIgDZM#p51xQvMr^46|XTOAgxE4np zp39dx+>Lt9_#Rz-=bdCk)FlCqnU&}2ZS3LYha@$0rp_e2 zF;ib3wHjU$lU^CBzK31nBRidVMSN&t)qQ3nCFUqe@FY?@ZYspXI%r+_e7-|P&+91H zyXG4>y)EYphh8|{MN#vgJ*UjypjF{iGi?~!|M4)>nKim17q9u^cAh>Z+7ly-T%|3p z7_E|K|MvRheAV}CjL&}?OGc+vNFAOn(jQ%~f-{#{i)V<+e2ghhK8|Cx6!Fax2`6ys z!4Pu@mxy5OIZ@%1EL9c8JD)z~EvnhCMotT`OyDItE;kXT=QOUPzbx_Dd0p8sPiK_0 z!DxwX436!>oP;9d>FhN3{1RGrETwWHx()iSh zsnO)6#I2S)9L3I_ykSEL5+k9DLcQx>cIb2jm>wR@yElY;Fi7W_O*1Hex37AiZw^Gh z*#b;^9QnaB3V@6Vvc2o>pXm8pG95(M>ew89(9gluXh!CY=2CrDB{#jv@5}^6|4|)(f0HoU`?COWwBmn2m_3E+DG=#~0T6o;^FfaQs(=bXiJ%ZIx4+Im zk@Js5y#5Wq|9*Y`L99^Qq~^3*pR6*Z04U6Dbev!3RTYN}gE8|{z@op2W2eK^VfL8# zj_)Pfo8s>Q_FIAK5L9rGm$CaeHI(^t!s;nQYnbc%@M*Al#qsuBjKW^h&8FV+bUzIaajH?hKgGsx#WjC{k(VG|Ayd~W>Gw{GFz0ZT!>esrEXec zpc-Y3#T&xtYsFcX04{I>*p?IbHX~T;f?Af;6zU#_OStK(s;?-~G8Tz@PB4FKDtz|AdZH2f1wSn}1C3iJ>2qk^ z9uN#Cp`OQqMJCoy<8+F}V<`0`7sELVFVcU_%J^i7lH`l{(!4<@xiGp@CW~Y>Tw<$u(hL~}k{{Iq72oU&5 zEUrI@rHw{o{fViV0TT8DTEB8GIY0$~{|PN(zDP{Me&t;M4RHPwvziuD&}XL`<{b== z00Mx!{Az27uuK=f`o^Vk+tyd&CF(ZixSYPqV6G_IHh}K~SD^~qiq(@$Z=Uj+^Bdia zgY!u@;3r$}h8Kv$gcUeKyUvwWc1M?SYoN@{9q=ATxviKpwS-QSL9mw0iF7#Zr0ZM zD4Qf2$vTq~V8ZHMcd*0Utu7PC1;r(J&+|0LsifqH=e4O>S}Err2X@p}x83vLP=PQc zjt?sr%8-V}e#C1qr23Xy;WN-hZ2A<%6?6^_i2*J`2|vok1c zT`VJ+Lk{pEEgF1H*^_w+H| zB|4YN#o@KfmsRuBr%2{?h2v^-kcTA0ny=g)boO46pJ<;s1|{hY0us;RMcu?r-Le-5;eOThqM`)>=zhJSK#g4{%iu2Ga*J;O_x=Er#6_Gr%!cx zbi7IHsIF5k>}vsUdvo&j>M$yi?2q&AvRORRA1BIccow+!nWai(meP-;|0lCZ0SJ*< z2Y~grJUR$0-IeRNH!t_#;nHUDB_;g1AG|y1{{>j*_bEeUlQ-b`Z?XxoKa0pF<$b37 zU)j$uF&ZHEr1&HE!T`EI>7)S=`9vV{DOzEFrT>*q|GJD9z__;z3`zwMsZItZK9c<9 zbb;VOAQE3f6XJyBl8O`I28K($mThdS$(sCOPDMeC)apgYGR_8p&oP|tD1osx> zIaxND4TEH*>r~b@zehayT(Gs_KP}W~4(U8TATj+#+Un`{=2is|loMa$IQ;Y(0(lj| zzmnl!pno^JC)cEe48j~BiX2`Fe{Ea3-Xu$1yu6`sWOWPD5mP#7q;*K~sE0eM>VAP$ z(rV#|`s?bcUGDMC0J{^Y1fR76Q}wSn%Ck}VKHZbBM8%WbU`P+Fs=|gqr{&U~CD2l1 z`_a<#sc5+uuHQ#cdNa(=L(o&;JJWrXv}oA4>co0&OvbMLvlA5@kB#%ub)d3*>zj^Z zvk$Jo@sJDv54lV%&H(}cl?vD2*Vzsf5DAg*0nR@vZa^0zG;lmNVL5!pnT9+7A^*e07POyVK`iV z>4u9eAA(v{5U3or!hRb=Pm+$Fen6U*g-Fhs1Wf+)0SecFYK2oIC}T^F^?EJt*Nhj9 zA--xUp2FDztC)A2Yp>L`gp5}`2dGsL&icxy6j~Urer_O<7s=8-{vs2*)8BZm{kQC% zSdN3@!6a;_⪚iW3DDi@Xl(!?A)rDY{d-DA@5Io%N=$A+Aeo^y2D!iOjK(j16S>)QuQHvgL%xF@#WD_S|l4cHl-MMTKRG9ikMw0^^|A>Ri`4e&oh*l)p4*SVewfVH*&4A+Muw8 zTk&?PXN#$JU(zx%mhL@^U$0cptul`;g^916*B{0^nx>DlR7o z`+R)S99pYGP8FDwbHlsR|Pdrm8f IWWDZx01j4BE&u=k literal 0 HcmV?d00001 diff --git a/tests/enrichers/test_timestamping_enricher.py b/tests/enrichers/test_timestamping_enricher.py index 706678d..22cab06 100644 --- a/tests/enrichers/test_timestamping_enricher.py +++ b/tests/enrichers/test_timestamping_enricher.py @@ -29,6 +29,12 @@ def selfsigned_response() -> TimeStampResponse: return decode_timestamp_response(f.read()) +@pytest.fixture +def digicert_response() -> TimeStampResponse: + with open("tests/data/timestamping/digicert.tsr", "rb") as f: + return f.read() + + @pytest.fixture def filehash(): return "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef" @@ -65,7 +71,6 @@ def test_full_enriching_selfsigned(setup_module, sample_media, mocker, selfsigne # set self-signed on tsp tsp.allow_selfsigned = True - tsp.enrich(metadata) assert len(metadata.media) @@ -131,12 +136,15 @@ def test_full_enriching_multiple_tsa(setup_module, sample_media, mocker, timesta assert len(timestamp_token_media.get("cert_chain")) == 3 -@pytest.mark.download -def test_fails_for_digicert(setup_module): +def test_fails_for_digicert(setup_module, mocker, digicert_response): """ Digicert TSRs are not compliant with RFC 3161. See https://github.com/trailofbits/rfc3161-client/issues/104#issuecomment-2621960840 """ + mocker.patch("requests.sessions.Session.post", return_value=requests.Response()) + mocker.patch("requests.Response.raise_for_status") + mocker.patch("requests.Response.content", new_callable=mocker.PropertyMock, return_value=digicert_response) + tsa_url = "http://timestamp.digicert.com" tsp: TimestampingEnricher = setup_module("timestamping_enricher") @@ -191,16 +199,10 @@ def test_order_crt_correctly(setup_module, wrong_order_timestamp_response): assert ordered_certs[1].subject.rfc4514_string() == "CN=TrustID Timestamping CA 3,O=IdenTrust,C=US" -def test_invalid_tsa_404(setup_module, mocker): - tsp = setup_module("timestamping_enricher") - post_mock = mocker.patch("requests.sessions.Session.post") - post_mock.side_effect = Exception("error") - with pytest.raises(Exception, match="error"): - tsp.sign_data("http://bellingcat.com/", b"my-message") - - -@pytest.mark.download def test_invalid_tsa_invalid_response(setup_module, mocker): + mocker.patch("requests.sessions.Session.post", return_value=requests.Response()) + raise_for_status = mocker.patch("requests.Response.raise_for_status") + raise_for_status.side_effect = requests.exceptions.HTTPError("404 Client Error") tsp = setup_module("timestamping_enricher") with pytest.raises(requests.exceptions.HTTPError, match="404 Client Error"): From 31fa7380f559de2fd5ac13a720dd10306e3d8581 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Mon, 24 Mar 2025 16:00:40 +0400 Subject: [PATCH 15/20] Fix up unit tests + issue when working with self-signed certs --- .../timestamping_enricher.py | 7 +++++-- tests/data/timestamping/self_signed.tsr | Bin 1600 -> 2905 bytes tests/enrichers/test_timestamping_enricher.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index 93d3ae8..385787b 100644 --- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -83,6 +83,7 @@ class TimestampingEnricher(Enricher): # fail if there's any issue with the certificates, uses certifi list of trusted CAs or the user-defined `cert_authorities` root_cert = self.verify_signed(signed, message) + if not root_cert: if self.allow_selfsigned: logger.warning(f"Allowing self-signed certificat from TSA {tsa_url=}") @@ -168,7 +169,6 @@ class TimestampingEnricher(Enricher): return certificate except Rfc3161VerificationError as e: continue - return None def sign_data(self, tsa_url: str, bytes_data: bytes) -> TimeStampResponse: @@ -216,8 +216,11 @@ class TimestampingEnricher(Enricher): def save_certificate(self, tsp_response: TimeStampResponse, verified_root_cert: x509.Certificate) -> list[Media]: # returns the leaf certificate URL, fails if not set - certificates = self.tst_certs(tsp_response) + [verified_root_cert] + certificates = self.tst_certs(tsp_response) + if verified_root_cert: + # add the verified root certificate (if there is one - self signed certs will have None here) + certificates += [verified_root_cert] cert_chain = [] for i, cert in enumerate(certificates): diff --git a/tests/data/timestamping/self_signed.tsr b/tests/data/timestamping/self_signed.tsr index e7ffd8344a4476af45816719931c4864d7bc8b1f..f78400eeb4ee3f7932e84a78043f4b57b9cc3aae 100644 GIT binary patch literal 2905 zcmd5;do)!09^ZSm=QuFjY6fw$mB7Y3BB!6PF}6j(vedh zSLj4k^i*6rsV+*PR!TzYp>B0kI6d6GJ3VyPy8qpE*4=CE{rkRtzwhV!TfguAeSnG{ z04N9y0TtbcN}HADnKr_J)sZ326zSn!&Hsc_#Bvc)WG6DO+)9uk1vA?<;BCZtCj0$En!Z*+YCEn2hDeH-;yKD zGOrkZOJ@wcZt81`9%fu>a2wF@zbxu$ndR}|V*e<0vUzJbf=;dKkJe}M0pAV?__i1( zaK!9`0h)rq!!SIKXspkesc+f*tf5Gy-UL+Y^+2UoRz#GCAS@XKK~&EgO7r}MEu(}? zXe7Wdb-)a=t`y7wu&I;)RE?!0^jpZ~b1?w8z{yd7VHh3INEelb2>n0r3884RsJ>o8 zFNp+W06OWVvrv(6fe5n)wxmj7afPlNKe3OO#ETG?xcm6Hio_g|J4fQ>?&~M!a5+Np z5{WCvpL8ekfhn88w+QU~5U15OBQ~`MGlv1waK)o48l7 z8bndSOybPInurR9p`Kpf{r*YLSLkNrtzAF6yfZZ5X!~y$&9*Fgf2Y()(mC`{Bi5RA z^jOH0mm3=A+oX;M>R0b9N};Y34R<9p!cIoYc1N=UuZJ z6I$>3Tivi+&Pjv(vb)gH+ z9bv9A8{5}E3HdED;@$!Em`^Wb0-xrv>o0sb-S7WTh36izIEP-G`^!nYZlg!anB-$T z3y$j$eHg7qwz}qWgvoYshDw+CT;Jl&-5b-l9*G?tz8#%N_qZ<#4yai(RFPA}lc14h5I=T4q$C1a zLQH75Y^_Wg8f~*yPQpx(#>o{jr943<=Luac`QU475#pETA|#PJ@%f{{2!TtypHDn7 zl5Iu|iF{(Q=(P9=$zo(-GUb;n6BGY$QxYAc@pV&S7{%WMd=w;owE*O=tX~PXOi28f z?}ZX*ktcvf9w`Js6aO#7znG#T?j&y{q-k_X;WXX4q*3iBLP6nFbK0wEK6dx->tGuw zyI*fT$&R!RtLU9^ZHGm;dQ;MhP&cD!gB>A{^H(^Yn1Om-HpK?6NFz=e16M``){Ad+ zcryzXz5bnb<%M;_vUTOUw< znd}44#D(ijI@ZCr=a!oXWsD7sVCki5#~Xb-#ua;azk6`Err+4hv`~H1+%mOGsz`+2;*T5wU3U;QdX-{aYUX|`jf>JJ;UsA@AU5B1I~T(R(JDm zDSB&nO*i*mHCAM}(kSPpR&CH`y${_*gKsYNvh{eCSz+Bbqp6vmX9dRlbD0&Htj?;6 zjWPX$2I(V_k~i)l+plHY9nbGd!a-(+29MtAY-uk+H(W-O0b zo>x-bwhu}Fo$Ye#edNo2qX)i^=U7-a5^S6lR(P{MsO7z1NkC!!@sS~ir1K+vB|TyKJYHglMP1nw`jdN& ziVOpj>cvg3D7{U?%VLW!IET@={E!w*g`5E@q?X_n0MGkBxSszjiGNKd+bB>qf^D> zU7;Q}=0~UHIc`y`y+9NF;FI61mRldGl9nKBLy>`IRa)@nUfI<$>5VfM zOY;(Eix}tUBBdvSGRwu@x)Yn1AP1-lhPDrPy)`zB>4;o&i=+Fb@?mi~%}x8!wug7m z8xB`yz}L?l$*n@?1{dRAr(fjN>GVXLx8_G^c!XNsyYx#%iZA!w!GwK5Go!6i%YM7K zCf99g!=);TO<;hPQukbI+|F@0eA4~tAD*-P;*S)Zy-~X~x$oxRB@)!<>V{)RMIv*i z!K*ttNxvGhWA)86x0tUoY1(wf^sMoUEJyaP&eEQ=r+T_tjEo&d=Ulc#o$9fOEqB)V zSl;^hIDc@j$1<-3Uulyx_KtU!wA3bQtt_@P=P-TadYAOL!u{#nRb8{!rRpk~HfBch gn&zO!!)(=Zd>-YFn<6p5U9ofd&Yoq=z{l(V1HAFyKL7v# literal 1600 zcmbW1X;71A5XbYrZxRwn2*^1o5{VWGf=>wHQ0@k#BBIElI4BsRBnSwhq*$yZ#T3EH zA&RzxNjNluNI)4u@F=K+dlV?(2q;mmQVEL6q0-oKDt_n}KR(a?XJ_}n`&%HyIs*e3 zLIE9uw*Mh^LPR1bE|%;c9gz%#*i~3ORpcXjON2CFi4eO?{iO{<2#o}YSbVt12Vx^= z7{Z`{pa84&Nr~5i&=SF~Xh8-L`~pJ|7NUafOZ5l_g?3+vzkm|QNoO`zBy#bsET^HP zCf%_klA}XgFp0*#HA>O;~sl){|F$Y?yo1Qcd&tjKR+9F|{#GrQ_ z$T3Id+KX1CX%BaqG07I#G60=MquJTfY#l>kJnD_($!9t1YQrQ#!~+Ns7a)Y!NDvr; zVO><3@7v{&#oba;k*TbTv$O7FiJw}4wA3xx(@cOd770RY5_I`|z73CaAa+0JAe(2y zii@Ec1H+Fogh296V6(}AJ3Poc*a=DO1hCSGNOJ@XIsoP>1yVQX*VuPr?g*P^gdWWIoc+jM1b(#Fxfo(?J zITHcv9%XYn+{;ZBxCb(Rz2Zxkd$}PG?lCQ-0eX~Ig~wk#G4}|+ zUeiN4d^^)w>YVtn%9JQoQl%|AJigF(x9jQyY5T(F@6>K*s(LE5OXX#uIC9Vjx;I_r zMZlfkOY*Dmb5#s#Ql@K5B8wa=E~OS1dRevx`ri*#bWdHGi8=O8qp2bxkKf2-;Lmh& ze;kwTE^Sq0=)wquf{MHW6KJU?V@g7!5m+Pj-!Yl4fuKb+0cc`zR#*sw(Liv(^rJ`{ zZHU&d&fXQwis*0TraY|E3+~d~oSHtk)M@oHsGWvT&Q0m=CDuW~q2(<$devhduV`63 z-8pUpN@qhe;uKf7s_$O-7)yM|!In2WcH(()wYkF4PaL1_+w#I_F1M_3yHq!FwMIbk z9=n(|izFgO4NKm5_3~r0l$~qGeh^oh()$m|b$&WXwch3Nl2L=j z-p;+#JifW@G1hH(ADe&orOaxT zY{eXdkz!Z^XswRwi%UwPAFwzN^=hCWLj>01?$p1K+P*8DzfO{n+@q<1|Gj2sd$mKRNPbYjBCY#CRfY zxGpFuecera&e_^%!iN?M!5?7_TNZOWwe*#zCqrhTYh>oc5aak`$xApeJB^@dG}CXVB+olT361ye4qDbx|Q*~qcivUKVj)=Q2+n{ diff --git a/tests/enrichers/test_timestamping_enricher.py b/tests/enrichers/test_timestamping_enricher.py index 22cab06..9e67ffd 100644 --- a/tests/enrichers/test_timestamping_enricher.py +++ b/tests/enrichers/test_timestamping_enricher.py @@ -73,7 +73,7 @@ def test_full_enriching_selfsigned(setup_module, sample_media, mocker, selfsigne tsp.allow_selfsigned = True tsp.enrich(metadata) - assert len(metadata.media) + assert len(metadata.media) == 2 def test_full_enriching(setup_module, sample_media, mocker, timestamp_response, filehash): From 3c4625d7086df1bd51738b7d0e249bd39857ccc8 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Mon, 24 Mar 2025 16:39:59 +0400 Subject: [PATCH 16/20] Further ruff tweaks --- .../timestamping_enricher/timestamping_enricher.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index 385787b..680538b 100644 --- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -7,12 +7,7 @@ from slugify import slugify import requests from loguru import logger -from rfc3161_client import ( - TimestampRequestBuilder, - TimeStampResponse, - decode_timestamp_response, - VerifierBuilder -) +from rfc3161_client import (decode_timestamp_response,TimestampRequestBuilder,TimeStampResponse, VerifierBuilder) from rfc3161_client import VerificationError as Rfc3161VerificationError from rfc3161_client.base import HashAlgorithm from rfc3161_client.tsp import SignedData @@ -167,8 +162,9 @@ class TimestampingEnricher(Enricher): try: verifier.verify(timestamp_response, message_hash) return certificate - except Rfc3161VerificationError as e: + except Rfc3161VerificationError: continue + return None def sign_data(self, tsa_url: str, bytes_data: bytes) -> TimeStampResponse: From 4af23e13d1a14cf8e3d34af1f1ca036cf83dbf3e Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Wed, 26 Mar 2025 14:38:21 +0400 Subject: [PATCH 17/20] Bump rfc3161-client to 1.0.1 --- poetry.lock | 32 ++++++++++++++++---------------- pyproject.toml | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7bca544..29bb299 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2255,25 +2255,25 @@ six = ">=1.7.0" [[package]] name = "rfc3161-client" -version = "1.0.0" +version = "1.0.1" description = "" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "rfc3161_client-1.0.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:702d83cd6f4bcaf648ea20c5bc64e69ca8ecc99e49dd6ecc82b574739d4792f1"}, - {file = "rfc3161_client-1.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:d4474600be7aa12d37838bff6d218326b9b08059ee4afa757ce16151f168b76a"}, - {file = "rfc3161_client-1.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a3374d72945bae12ceac2a68e39953ed95ed2a35121cd5f3d26f90851f53c26"}, - {file = "rfc3161_client-1.0.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:93c56d7de2f43be6aa4a1d1c8723982453517b77512e9b6dd46cc2343c92b3a5"}, - {file = "rfc3161_client-1.0.0-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:311ae7278cf914b448ebf707be566161025c0b26180629c78fb0efb5863edd3a"}, - {file = "rfc3161_client-1.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f46be2701f60d6649e0ff4bb094e0e0056f934324f7e3e1d2ac811964a171c1b"}, - {file = "rfc3161_client-1.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f88e19cb92fcf9eb67494d006b4bd3c7a206cb1c6b207b6bd3b56bea767caa7f"}, - {file = "rfc3161_client-1.0.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:f0fda030a58ba1840aecf56d234458cd7ace7437d4a03d650dcbbfbe328528fc"}, - {file = "rfc3161_client-1.0.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:c593b0a990ee48f3fb2f92dd5a6c41b81930c0a7749161aff5c6f173a3ce75ec"}, - {file = "rfc3161_client-1.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:67dd79572d7cf6c658acd0d0bde2892bf0a5b6ec22f68156f23ef117bee27bc8"}, - {file = "rfc3161_client-1.0.0-cp39-abi3-win32.whl", hash = "sha256:90e5969b82e0939821c06b2adb4322c6a55337337a169df7ecfcb808c7c2a14e"}, - {file = "rfc3161_client-1.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:1dbc6585957dadcfec531779c48f15e51242c91e94082b99f56a8426cacd1974"}, - {file = "rfc3161_client-1.0.0.tar.gz", hash = "sha256:626619d12154c793c034fc72fdb05043b69cb4ca278cdb0b889194e87af7172b"}, + {file = "rfc3161_client-1.0.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:75d8c9d255fa79b9ae4aa27cee519893599efd79f9e6c24a1194dd296ce1c210"}, + {file = "rfc3161_client-1.0.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0d3db059fe08d8b6b06aff89e133fcc352ffea1a1dafadb116dda9dae59d0689"}, + {file = "rfc3161_client-1.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdef0c9d3213ca5b79d7f76ada48ae10c5011cb25abed2f6df07b344d16d1c28"}, + {file = "rfc3161_client-1.0.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7c34ce4d7d2bf5207c54de3a771e757f1f8bb04a8469d3cef6aefe074841064d"}, + {file = "rfc3161_client-1.0.1-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4809f2fcfb5f8b42261a7b831929f62a297b584c8d1f4d242eae5e9447674b6"}, + {file = "rfc3161_client-1.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a644b220b7f0f0be7856f49b043651982bd76e7aa9eb17b3e4e303fde36ed5a1"}, + {file = "rfc3161_client-1.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bb03a5a77b07adf766b7daac6cb8b7a8337ffc8f6d6046af74469973f52df8e1"}, + {file = "rfc3161_client-1.0.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:d6c6e4626780b1c531d32d6a126d6c27865b1eb59c65e8b0f1f8f94aa3205285"}, + {file = "rfc3161_client-1.0.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:912c2f049ce23d0f1c173b6fbd8673f964a27ad97907064dbc74f86dd0d95d15"}, + {file = "rfc3161_client-1.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:081211a1b602b6dff7feb314d39ca2229c8db4e8cf55eef0c35b460470f4b2bb"}, + {file = "rfc3161_client-1.0.1-cp39-abi3-win32.whl", hash = "sha256:59efa8fddf72a15e397276fe512dbfb99c0dc95032b495815bfc4f8f16302f2c"}, + {file = "rfc3161_client-1.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:5381a63d5ed5b3c257cb18aacf3f737b1a1ad6df634290fe689b6d601c61cd24"}, + {file = "rfc3161_client-1.0.1.tar.gz", hash = "sha256:1c951f3912b90c6d3f3505e644b74ee08543387253647b86459addbffb16f63f"}, ] [package.dependencies] @@ -2281,7 +2281,7 @@ cryptography = ">=43,<45" [package.extras] dev = ["maturin (>=1.7,<2.0)", "rfc3161-client[doc,lint,test]"] -lint = ["interrogate", "ruff (>=0.7,<0.9)"] +lint = ["interrogate", "ruff (>=0.7,<0.12)"] test = ["coverage[toml]", "pretend", "pytest", "pytest-cov"] [[package]] @@ -3433,4 +3433,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "b150860980a1d0d707ac439f2a1696c12230abb9b11bda28678edac758cbaff9" +content-hash = "82094ed69f775426a5043b8ed46b580ac12fc298c8796a8099a77c2d2da58e73" diff --git a/pyproject.toml b/pyproject.toml index 69b653f..2705554 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ dependencies = [ "certvalidator (>=0.0.0)", "rich-argparse (>=1.6.0,<2.0.0)", "ruamel-yaml (>=0.18.10,<0.19.0)", - "rfc3161-client (>=1.0.0,<2.0.0)", + "rfc3161-client (>=1.0.1,<2.0.0)", "opentimestamps (>=0.4.5,<0.5.0)", ] From 46e31808f6b3481d5c0bccbc068ef3960452fa2a Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Wed, 26 Mar 2025 14:42:29 +0400 Subject: [PATCH 18/20] Version bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2705554..07f375d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [project] name = "auto-archiver" -version = "0.13.8" +version = "0.13.9" description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)." requires-python = ">=3.10,<3.13" From 0073a08525569b1de2316e61902066c85ed567fb Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Wed, 26 Mar 2025 14:57:55 +0400 Subject: [PATCH 19/20] Update manifest dependencies to remove tsp_client et al. --- src/auto_archiver/modules/timestamping_enricher/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py index ee67b8d..403e0ef 100644 --- a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py +++ b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py @@ -3,7 +3,7 @@ "type": ["enricher"], "requires_setup": True, "dependencies": { - "python": ["loguru", "slugify", "tsp_client", "asn1crypto", "certvalidator", "certifi"], + "python": ["loguru", "slugify", "cryptography", "rfc3161_client", "certifi"], }, "configs": { "tsa_urls": { From cb3ae055d60f3097ceec7e217a7ff9f24b1d58b5 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Wed, 26 Mar 2025 15:11:25 +0400 Subject: [PATCH 20/20] Also remove certvalidator from poetry/project --- poetry.lock | 45 +-------------------------------------------- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 45 deletions(-) diff --git a/poetry.lock b/poetry.lock index 29bb299..cd602b1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -54,18 +54,6 @@ doc = ["Sphinx (>=8.2,<9.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", test = ["anyio[trio]", "blockbuster (>=1.5.23)", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1) ; python_version >= \"3.10\"", "uvloop (>=0.21) ; platform_python_implementation == \"CPython\" and platform_system != \"Windows\" and python_version < \"3.14\""] trio = ["trio (>=0.26.1)"] -[[package]] -name = "asn1crypto" -version = "1.5.1" -description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"}, - {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, -] - [[package]] name = "astroid" version = "3.3.9" @@ -385,22 +373,6 @@ files = [ {file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"}, ] -[[package]] -name = "certvalidator" -version = "0.11.1" -description = "Validates X.509 certificates and paths" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "certvalidator-0.11.1-py2.py3-none-any.whl", hash = "sha256:77520b269f516d4fb0902998d5bd0eb3727fe153b659aa1cb828dcf12ea6b8de"}, - {file = "certvalidator-0.11.1.tar.gz", hash = "sha256:922d141c94393ab285ca34338e18dd4093e3ae330b1f278e96c837cb62cffaad"}, -] - -[package.dependencies] -asn1crypto = ">=0.18.1" -oscrypto = ">=0.16.1" - [[package]] name = "cffi" version = "1.17.1" @@ -1450,21 +1422,6 @@ files = [ pycryptodomex = ">=3.3.1" python-bitcoinlib = ">=0.9.0,<0.13.0" -[[package]] -name = "oscrypto" -version = "1.3.0" -description = "TLS (SSL) sockets, key generation, encryption, decryption, signing, verification and KDFs using the OS crypto libraries. Does not require a compiler, and relies on the OS for patching. Works on Windows, OS X and Linux/BSD." -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "oscrypto-1.3.0-py2.py3-none-any.whl", hash = "sha256:2b2f1d2d42ec152ca90ccb5682f3e051fb55986e1b170ebde472b133713e7085"}, - {file = "oscrypto-1.3.0.tar.gz", hash = "sha256:6f5fef59cb5b3708321db7cca56aed8ad7e662853351e7991fcf60ec606d47a4"}, -] - -[package.dependencies] -asn1crypto = ">=1.5.1" - [[package]] name = "outcome" version = "1.3.0.post0" @@ -3433,4 +3390,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "82094ed69f775426a5043b8ed46b580ac12fc298c8796a8099a77c2d2da58e73" +content-hash = "697ef9d5a7ac2f5bfb8014a52fb068732c717f4a502eaba7e20b26d0912e48e2" diff --git a/pyproject.toml b/pyproject.toml index 07f375d..6b34c2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,6 @@ dependencies = [ "jsonlines (>=0.0.0)", "pysubs2 (>=0.0.0)", "retrying (>=0.0.0)", - "certvalidator (>=0.0.0)", "rich-argparse (>=1.6.0,<2.0.0)", "ruamel-yaml (>=0.18.10,<0.19.0)", "rfc3161-client (>=1.0.1,<2.0.0)",