pull/247/head
Patrick Robertson 2025-03-14 12:38:12 +00:00
rodzic b8da7607e8
commit 17ae75fb95
3 zmienionych plików z 109 dodań i 88 usunięć

Wyświetl plik

@ -18,12 +18,12 @@
], ],
"help": "List of OpenTimestamps calendar servers to use for timestamping. See here for a list of calendars maintained by opentimestamps:\ "help": "List of OpenTimestamps calendar servers to use for timestamping. See here for a list of calendars maintained by opentimestamps:\
https://opentimestamps.org/#calendars", https://opentimestamps.org/#calendars",
"type": "list" "type": "list",
}, },
"calendar_whitelist": { "calendar_whitelist": {
"default": [], "default": [],
"help": "Optional whitelist of calendar servers. Override this if you are using your own calendar servers. e.g. ['https://mycalendar.com']", "help": "Optional whitelist of calendar servers. Override this if you are using your own calendar servers. e.g. ['https://mycalendar.com']",
"type": "list" "type": "list",
}, },
}, },
"description": """ "description": """
@ -96,5 +96,5 @@ Calendar https://alice.btc.calendar.opentimestamps.org: Timestamped by transacti
if you want to use your own calendars, then you can override this setting in the `calendar_whitelist` configuration option. if you want to use your own calendars, then you can override this setting in the `calendar_whitelist` configuration option.
""" """,
} }

Wyświetl plik

@ -11,8 +11,8 @@ from auto_archiver.core import Enricher
from auto_archiver.core import Metadata, Media from auto_archiver.core import Metadata, Media
from auto_archiver.utils.misc import get_current_timestamp from auto_archiver.utils.misc import get_current_timestamp
class OpentimestampsEnricher(Enricher):
class OpentimestampsEnricher(Enricher):
def enrich(self, to_enrich: Metadata) -> None: def enrich(self, to_enrich: Metadata) -> None:
url = to_enrich.get_url() url = to_enrich.get_url()
logger.debug(f"OpenTimestamps timestamping files for {url=}") logger.debug(f"OpenTimestamps timestamping files for {url=}")
@ -37,7 +37,7 @@ class OpentimestampsEnricher(Enricher):
# SHA256 is the recommended hash, ref: https://github.com/bellingcat/auto-archiver/pull/247#discussion_r1992433181 # SHA256 is the recommended hash, ref: https://github.com/bellingcat/auto-archiver/pull/247#discussion_r1992433181
logger.debug(f"Creating timestamp for {file_path}") logger.debug(f"Creating timestamp for {file_path}")
file_hash = None file_hash = None
with open(file_path, 'rb') as f: with open(file_path, "rb") as f:
file_hash = OpSHA256().hash_fd(f) file_hash = OpSHA256().hash_fd(f)
if not file_hash: if not file_hash:
@ -79,14 +79,16 @@ class OpentimestampsEnricher(Enricher):
# If all calendar submissions failed, add pending attestations # If all calendar submissions failed, add pending attestations
if not submitted_to_calendar and not timestamp.attestations: if not submitted_to_calendar and not timestamp.attestations:
logger.error(f"Failed to submit to any calendar for {file_path}. **This file will not be timestamped.**") logger.error(
f"Failed to submit to any calendar for {file_path}. **This file will not be timestamped.**"
)
media.set("opentimestamps", False) media.set("opentimestamps", False)
continue continue
# Save the timestamp proof to a file # Save the timestamp proof to a file
timestamp_path = os.path.join(self.tmp_dir, f"{os.path.basename(file_path)}.ots") timestamp_path = os.path.join(self.tmp_dir, f"{os.path.basename(file_path)}.ots")
try: try:
with open(timestamp_path, 'wb') as f: with open(timestamp_path, "wb") as f:
# Create a serialization context and write to the file # Create a serialization context and write to the file
ctx = serialize.BytesSerializationContext() ctx = serialize.BytesSerializationContext()
detached_timestamp.serialize(ctx) detached_timestamp.serialize(ctx)

Wyświetl plik

@ -1,7 +1,4 @@
from pathlib import Path
import pytest import pytest
import os
import tempfile
import hashlib import hashlib
from opentimestamps.core.timestamp import Timestamp, DetachedTimestampFile from opentimestamps.core.timestamp import Timestamp, DetachedTimestampFile
@ -26,11 +23,13 @@ def sample_file_path(tmp_path):
tmp_file.write_text("This is a test file content for OpenTimestamps") tmp_file.write_text("This is a test file content for OpenTimestamps")
return str(tmp_file) return str(tmp_file)
@pytest.fixture @pytest.fixture
def detached_timestamp_file(): def detached_timestamp_file():
"""Create a simple detached timestamp file for testing""" """Create a simple detached timestamp file for testing"""
file_hash = hashlib.sha256(b"Test content").digest() file_hash = hashlib.sha256(b"Test content").digest()
from opentimestamps.core.op import OpSHA256 from opentimestamps.core.op import OpSHA256
file_hash_op = OpSHA256() file_hash_op = OpSHA256()
timestamp = Timestamp(file_hash) timestamp = Timestamp(file_hash)
@ -44,11 +43,13 @@ def detached_timestamp_file():
return DetachedTimestampFile(file_hash_op, timestamp) return DetachedTimestampFile(file_hash_op, timestamp)
@pytest.fixture @pytest.fixture
def verified_timestamp_file(): def verified_timestamp_file():
"""Create a timestamp file with a Bitcoin attestation""" """Create a timestamp file with a Bitcoin attestation"""
file_hash = hashlib.sha256(b"Verified content").digest() file_hash = hashlib.sha256(b"Verified content").digest()
from opentimestamps.core.op import OpSHA256 from opentimestamps.core.op import OpSHA256
file_hash_op = OpSHA256() file_hash_op = OpSHA256()
timestamp = Timestamp(file_hash) timestamp = Timestamp(file_hash)
@ -58,11 +59,13 @@ def verified_timestamp_file():
return DetachedTimestampFile(file_hash_op, timestamp) return DetachedTimestampFile(file_hash_op, timestamp)
@pytest.fixture @pytest.fixture
def pending_timestamp_file(): def pending_timestamp_file():
"""Create a timestamp file with only pending attestations""" """Create a timestamp file with only pending attestations"""
file_hash = hashlib.sha256(b"Pending content").digest() file_hash = hashlib.sha256(b"Pending content").digest()
from opentimestamps.core.op import OpSHA256 from opentimestamps.core.op import OpSHA256
file_hash_op = OpSHA256() file_hash_op = OpSHA256()
timestamp = Timestamp(file_hash) timestamp = Timestamp(file_hash)
@ -74,15 +77,15 @@ def pending_timestamp_file():
return DetachedTimestampFile(file_hash_op, timestamp) return DetachedTimestampFile(file_hash_op, timestamp)
@pytest.mark.download @pytest.mark.download
def test_download_tsr(setup_module, mocker): def test_download_tsr(setup_module, mocker):
"""Test submitting a hash to calendar servers""" """Test submitting a hash to calendar servers"""
# Mock the RemoteCalendar submit method # Mock the RemoteCalendar submit method
mock_submit = mocker.patch.object(RemoteCalendar, 'submit') mock_submit = mocker.patch.object(RemoteCalendar, "submit")
test_timestamp = Timestamp(hashlib.sha256(b"test").digest()) test_timestamp = Timestamp(hashlib.sha256(b"test").digest())
mock_submit.return_value = test_timestamp mock_submit.return_value = test_timestamp
ots = setup_module("opentimestamps_enricher") ots = setup_module("opentimestamps_enricher")
# Create a calendar # Create a calendar
@ -96,6 +99,7 @@ def test_download_tsr(setup_module, mocker):
assert isinstance(result, Timestamp) assert isinstance(result, Timestamp)
assert result == test_timestamp assert result == test_timestamp
def test_verify_timestamp(setup_module, detached_timestamp_file): def test_verify_timestamp(setup_module, detached_timestamp_file):
"""Test the verification of timestamp attestations""" """Test the verification of timestamp attestations"""
ots = setup_module("opentimestamps_enricher") ots = setup_module("opentimestamps_enricher")
@ -117,6 +121,7 @@ def test_verify_timestamp(setup_module, detached_timestamp_file):
bitcoin_attestation = next(a for a in verification_info["attestations"] if a["status"] == "confirmed") bitcoin_attestation = next(a for a in verification_info["attestations"] if a["status"] == "confirmed")
assert bitcoin_attestation["block_height"] == 783000 assert bitcoin_attestation["block_height"] == 783000
def test_verify_pending_only(setup_module, pending_timestamp_file): def test_verify_pending_only(setup_module, pending_timestamp_file):
"""Test verification of timestamps with only pending attestations""" """Test verification of timestamps with only pending attestations"""
ots = setup_module("opentimestamps_enricher") ots = setup_module("opentimestamps_enricher")
@ -134,6 +139,7 @@ def test_verify_pending_only(setup_module, pending_timestamp_file):
assert "https://example1.calendar.com" in uris assert "https://example1.calendar.com" in uris
assert "https://example2.calendar.com" in uris assert "https://example2.calendar.com" in uris
def test_verify_bitcoin_completed(setup_module, verified_timestamp_file): def test_verify_bitcoin_completed(setup_module, verified_timestamp_file):
"""Test verification of timestamps with completed Bitcoin attestations""" """Test verification of timestamps with completed Bitcoin attestations"""
@ -150,11 +156,12 @@ def test_verify_bitcoin_completed(setup_module, verified_timestamp_file):
assert attestation["status"] == "confirmed" assert attestation["status"] == "confirmed"
assert attestation["block_height"] == 783000 assert attestation["block_height"] == 783000
def test_full_enriching(setup_module, sample_file_path, sample_media, mocker): def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
"""Test the complete enrichment process""" """Test the complete enrichment process"""
# Mock the calendar submission to avoid network requests # Mock the calendar submission to avoid network requests
mock_calendar = mocker.patch.object(RemoteCalendar, 'submit') mock_calendar = mocker.patch.object(RemoteCalendar, "submit")
# Create a function that returns a new timestamp for each call # Create a function that returns a new timestamp for each call
def side_effect(digest): def side_effect(digest):
@ -197,19 +204,30 @@ def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
assert timestamp_media.get("verified") == True assert timestamp_media.get("verified") == True
assert timestamp_media.get("attestation_count") == 1 assert timestamp_media.get("attestation_count") == 1
def test_full_enriching_one_calendar_error(setup_module, sample_file_path, sample_media, mocker, pending_timestamp_file):
def test_full_enriching_one_calendar_error(
setup_module, sample_file_path, sample_media, mocker, pending_timestamp_file
):
"""Test enrichment when one calendar server returns an error""" """Test enrichment when one calendar server returns an error"""
# Mock the calendar submission to raise an exception # Mock the calendar submission to raise an exception
mock_calendar = mocker.patch.object(RemoteCalendar, 'submit') mock_calendar = mocker.patch.object(RemoteCalendar, "submit")
test_timestamp = Timestamp(bytes.fromhex("583988e03646c26fa290c5c2408540a2f4e2aa9be087aa4546aefb531385b935")) test_timestamp = Timestamp(bytes.fromhex("583988e03646c26fa290c5c2408540a2f4e2aa9be087aa4546aefb531385b935"))
# Add a bitcoin attestation to the test timestamp # Add a bitcoin attestation to the test timestamp
bitcoin = BitcoinBlockHeaderAttestation(783000) bitcoin = BitcoinBlockHeaderAttestation(783000)
test_timestamp.attestations.add(bitcoin) test_timestamp.attestations.add(bitcoin)
mock_calendar.side_effect = [test_timestamp, Exception("Calendar server error")] mock_calendar.side_effect = [test_timestamp, Exception("Calendar server error")]
ots = setup_module("opentimestamps_enricher", {"calendar_urls": ["https://alice.btc.calendar.opentimestamps.org", "https://bob.btc.calendar.opentimestamps.org"]}) ots = setup_module(
"opentimestamps_enricher",
{
"calendar_urls": [
"https://alice.btc.calendar.opentimestamps.org",
"https://bob.btc.calendar.opentimestamps.org",
]
},
)
# Create test metadata with sample file # Create test metadata with sample file
metadata = Metadata().set_url("https://example.com") metadata = Metadata().set_url("https://example.com")
@ -221,15 +239,15 @@ def test_full_enriching_one_calendar_error(setup_module, sample_file_path, sampl
# Verify results # Verify results
assert metadata.get("opentimestamped") == True assert metadata.get("opentimestamped") == True
assert metadata.get("opentimestamps_count") == 1 # only alice worked, not bob assert metadata.get("opentimestamps_count") == 1 # only alice worked, not bob
def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_media, mocker): def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_media, mocker):
"""Test enrichment when calendar servers return errors""" """Test enrichment when calendar servers return errors"""
# Mock the calendar submission to raise an exception # Mock the calendar submission to raise an exception
mock_calendar = mocker.patch.object(RemoteCalendar, 'submit') mock_calendar = mocker.patch.object(RemoteCalendar, "submit")
mock_calendar.side_effect = Exception("Calendar server error") mock_calendar.side_effect = Exception("Calendar server error")
ots = setup_module("opentimestamps_enricher") ots = setup_module("opentimestamps_enricher")
# Create test metadata with sample file # Create test metadata with sample file
@ -244,6 +262,7 @@ def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_me
assert metadata.get("opentimestamped") == False assert metadata.get("opentimestamped") == False
assert metadata.get("opentimestamps_count") is None assert metadata.get("opentimestamps_count") is None
def test_no_files_to_stamp(setup_module): def test_no_files_to_stamp(setup_module):
"""Test enrichment with no files to timestamp""" """Test enrichment with no files to timestamp"""
ots = setup_module("opentimestamps_enricher") ots = setup_module("opentimestamps_enricher")