kopia lustrzana https://github.com/bellingcat/auto-archiver
Ruff fixes
rodzic
b8da7607e8
commit
17ae75fb95
|
@ -18,12 +18,12 @@
|
||||||
],
|
],
|
||||||
"help": "List of OpenTimestamps calendar servers to use for timestamping. See here for a list of calendars maintained by opentimestamps:\
|
"help": "List of OpenTimestamps calendar servers to use for timestamping. See here for a list of calendars maintained by opentimestamps:\
|
||||||
https://opentimestamps.org/#calendars",
|
https://opentimestamps.org/#calendars",
|
||||||
"type": "list"
|
"type": "list",
|
||||||
},
|
},
|
||||||
"calendar_whitelist": {
|
"calendar_whitelist": {
|
||||||
"default": [],
|
"default": [],
|
||||||
"help": "Optional whitelist of calendar servers. Override this if you are using your own calendar servers. e.g. ['https://mycalendar.com']",
|
"help": "Optional whitelist of calendar servers. Override this if you are using your own calendar servers. e.g. ['https://mycalendar.com']",
|
||||||
"type": "list"
|
"type": "list",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"description": """
|
"description": """
|
||||||
|
@ -96,5 +96,5 @@ Calendar https://alice.btc.calendar.opentimestamps.org: Timestamped by transacti
|
||||||
if you want to use your own calendars, then you can override this setting in the `calendar_whitelist` configuration option.
|
if you want to use your own calendars, then you can override this setting in the `calendar_whitelist` configuration option.
|
||||||
|
|
||||||
|
|
||||||
"""
|
""",
|
||||||
}
|
}
|
|
@ -11,8 +11,8 @@ from auto_archiver.core import Enricher
|
||||||
from auto_archiver.core import Metadata, Media
|
from auto_archiver.core import Metadata, Media
|
||||||
from auto_archiver.utils.misc import get_current_timestamp
|
from auto_archiver.utils.misc import get_current_timestamp
|
||||||
|
|
||||||
class OpentimestampsEnricher(Enricher):
|
|
||||||
|
|
||||||
|
class OpentimestampsEnricher(Enricher):
|
||||||
def enrich(self, to_enrich: Metadata) -> None:
|
def enrich(self, to_enrich: Metadata) -> None:
|
||||||
url = to_enrich.get_url()
|
url = to_enrich.get_url()
|
||||||
logger.debug(f"OpenTimestamps timestamping files for {url=}")
|
logger.debug(f"OpenTimestamps timestamping files for {url=}")
|
||||||
|
@ -37,7 +37,7 @@ class OpentimestampsEnricher(Enricher):
|
||||||
# SHA256 is the recommended hash, ref: https://github.com/bellingcat/auto-archiver/pull/247#discussion_r1992433181
|
# SHA256 is the recommended hash, ref: https://github.com/bellingcat/auto-archiver/pull/247#discussion_r1992433181
|
||||||
logger.debug(f"Creating timestamp for {file_path}")
|
logger.debug(f"Creating timestamp for {file_path}")
|
||||||
file_hash = None
|
file_hash = None
|
||||||
with open(file_path, 'rb') as f:
|
with open(file_path, "rb") as f:
|
||||||
file_hash = OpSHA256().hash_fd(f)
|
file_hash = OpSHA256().hash_fd(f)
|
||||||
|
|
||||||
if not file_hash:
|
if not file_hash:
|
||||||
|
@ -79,14 +79,16 @@ class OpentimestampsEnricher(Enricher):
|
||||||
|
|
||||||
# If all calendar submissions failed, add pending attestations
|
# If all calendar submissions failed, add pending attestations
|
||||||
if not submitted_to_calendar and not timestamp.attestations:
|
if not submitted_to_calendar and not timestamp.attestations:
|
||||||
logger.error(f"Failed to submit to any calendar for {file_path}. **This file will not be timestamped.**")
|
logger.error(
|
||||||
|
f"Failed to submit to any calendar for {file_path}. **This file will not be timestamped.**"
|
||||||
|
)
|
||||||
media.set("opentimestamps", False)
|
media.set("opentimestamps", False)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Save the timestamp proof to a file
|
# Save the timestamp proof to a file
|
||||||
timestamp_path = os.path.join(self.tmp_dir, f"{os.path.basename(file_path)}.ots")
|
timestamp_path = os.path.join(self.tmp_dir, f"{os.path.basename(file_path)}.ots")
|
||||||
try:
|
try:
|
||||||
with open(timestamp_path, 'wb') as f:
|
with open(timestamp_path, "wb") as f:
|
||||||
# Create a serialization context and write to the file
|
# Create a serialization context and write to the file
|
||||||
ctx = serialize.BytesSerializationContext()
|
ctx = serialize.BytesSerializationContext()
|
||||||
detached_timestamp.serialize(ctx)
|
detached_timestamp.serialize(ctx)
|
||||||
|
|
|
@ -1,7 +1,4 @@
|
||||||
from pathlib import Path
|
|
||||||
import pytest
|
import pytest
|
||||||
import os
|
|
||||||
import tempfile
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
|
||||||
from opentimestamps.core.timestamp import Timestamp, DetachedTimestampFile
|
from opentimestamps.core.timestamp import Timestamp, DetachedTimestampFile
|
||||||
|
@ -26,11 +23,13 @@ def sample_file_path(tmp_path):
|
||||||
tmp_file.write_text("This is a test file content for OpenTimestamps")
|
tmp_file.write_text("This is a test file content for OpenTimestamps")
|
||||||
return str(tmp_file)
|
return str(tmp_file)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def detached_timestamp_file():
|
def detached_timestamp_file():
|
||||||
"""Create a simple detached timestamp file for testing"""
|
"""Create a simple detached timestamp file for testing"""
|
||||||
file_hash = hashlib.sha256(b"Test content").digest()
|
file_hash = hashlib.sha256(b"Test content").digest()
|
||||||
from opentimestamps.core.op import OpSHA256
|
from opentimestamps.core.op import OpSHA256
|
||||||
|
|
||||||
file_hash_op = OpSHA256()
|
file_hash_op = OpSHA256()
|
||||||
timestamp = Timestamp(file_hash)
|
timestamp = Timestamp(file_hash)
|
||||||
|
|
||||||
|
@ -44,11 +43,13 @@ def detached_timestamp_file():
|
||||||
|
|
||||||
return DetachedTimestampFile(file_hash_op, timestamp)
|
return DetachedTimestampFile(file_hash_op, timestamp)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def verified_timestamp_file():
|
def verified_timestamp_file():
|
||||||
"""Create a timestamp file with a Bitcoin attestation"""
|
"""Create a timestamp file with a Bitcoin attestation"""
|
||||||
file_hash = hashlib.sha256(b"Verified content").digest()
|
file_hash = hashlib.sha256(b"Verified content").digest()
|
||||||
from opentimestamps.core.op import OpSHA256
|
from opentimestamps.core.op import OpSHA256
|
||||||
|
|
||||||
file_hash_op = OpSHA256()
|
file_hash_op = OpSHA256()
|
||||||
timestamp = Timestamp(file_hash)
|
timestamp = Timestamp(file_hash)
|
||||||
|
|
||||||
|
@ -58,11 +59,13 @@ def verified_timestamp_file():
|
||||||
|
|
||||||
return DetachedTimestampFile(file_hash_op, timestamp)
|
return DetachedTimestampFile(file_hash_op, timestamp)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def pending_timestamp_file():
|
def pending_timestamp_file():
|
||||||
"""Create a timestamp file with only pending attestations"""
|
"""Create a timestamp file with only pending attestations"""
|
||||||
file_hash = hashlib.sha256(b"Pending content").digest()
|
file_hash = hashlib.sha256(b"Pending content").digest()
|
||||||
from opentimestamps.core.op import OpSHA256
|
from opentimestamps.core.op import OpSHA256
|
||||||
|
|
||||||
file_hash_op = OpSHA256()
|
file_hash_op = OpSHA256()
|
||||||
timestamp = Timestamp(file_hash)
|
timestamp = Timestamp(file_hash)
|
||||||
|
|
||||||
|
@ -74,15 +77,15 @@ def pending_timestamp_file():
|
||||||
|
|
||||||
return DetachedTimestampFile(file_hash_op, timestamp)
|
return DetachedTimestampFile(file_hash_op, timestamp)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.download
|
@pytest.mark.download
|
||||||
def test_download_tsr(setup_module, mocker):
|
def test_download_tsr(setup_module, mocker):
|
||||||
"""Test submitting a hash to calendar servers"""
|
"""Test submitting a hash to calendar servers"""
|
||||||
# Mock the RemoteCalendar submit method
|
# Mock the RemoteCalendar submit method
|
||||||
mock_submit = mocker.patch.object(RemoteCalendar, 'submit')
|
mock_submit = mocker.patch.object(RemoteCalendar, "submit")
|
||||||
test_timestamp = Timestamp(hashlib.sha256(b"test").digest())
|
test_timestamp = Timestamp(hashlib.sha256(b"test").digest())
|
||||||
mock_submit.return_value = test_timestamp
|
mock_submit.return_value = test_timestamp
|
||||||
|
|
||||||
|
|
||||||
ots = setup_module("opentimestamps_enricher")
|
ots = setup_module("opentimestamps_enricher")
|
||||||
|
|
||||||
# Create a calendar
|
# Create a calendar
|
||||||
|
@ -96,6 +99,7 @@ def test_download_tsr(setup_module, mocker):
|
||||||
assert isinstance(result, Timestamp)
|
assert isinstance(result, Timestamp)
|
||||||
assert result == test_timestamp
|
assert result == test_timestamp
|
||||||
|
|
||||||
|
|
||||||
def test_verify_timestamp(setup_module, detached_timestamp_file):
|
def test_verify_timestamp(setup_module, detached_timestamp_file):
|
||||||
"""Test the verification of timestamp attestations"""
|
"""Test the verification of timestamp attestations"""
|
||||||
ots = setup_module("opentimestamps_enricher")
|
ots = setup_module("opentimestamps_enricher")
|
||||||
|
@ -117,6 +121,7 @@ def test_verify_timestamp(setup_module, detached_timestamp_file):
|
||||||
bitcoin_attestation = next(a for a in verification_info["attestations"] if a["status"] == "confirmed")
|
bitcoin_attestation = next(a for a in verification_info["attestations"] if a["status"] == "confirmed")
|
||||||
assert bitcoin_attestation["block_height"] == 783000
|
assert bitcoin_attestation["block_height"] == 783000
|
||||||
|
|
||||||
|
|
||||||
def test_verify_pending_only(setup_module, pending_timestamp_file):
|
def test_verify_pending_only(setup_module, pending_timestamp_file):
|
||||||
"""Test verification of timestamps with only pending attestations"""
|
"""Test verification of timestamps with only pending attestations"""
|
||||||
ots = setup_module("opentimestamps_enricher")
|
ots = setup_module("opentimestamps_enricher")
|
||||||
|
@ -134,6 +139,7 @@ def test_verify_pending_only(setup_module, pending_timestamp_file):
|
||||||
assert "https://example1.calendar.com" in uris
|
assert "https://example1.calendar.com" in uris
|
||||||
assert "https://example2.calendar.com" in uris
|
assert "https://example2.calendar.com" in uris
|
||||||
|
|
||||||
|
|
||||||
def test_verify_bitcoin_completed(setup_module, verified_timestamp_file):
|
def test_verify_bitcoin_completed(setup_module, verified_timestamp_file):
|
||||||
"""Test verification of timestamps with completed Bitcoin attestations"""
|
"""Test verification of timestamps with completed Bitcoin attestations"""
|
||||||
|
|
||||||
|
@ -150,11 +156,12 @@ def test_verify_bitcoin_completed(setup_module, verified_timestamp_file):
|
||||||
assert attestation["status"] == "confirmed"
|
assert attestation["status"] == "confirmed"
|
||||||
assert attestation["block_height"] == 783000
|
assert attestation["block_height"] == 783000
|
||||||
|
|
||||||
|
|
||||||
def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
|
def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
|
||||||
"""Test the complete enrichment process"""
|
"""Test the complete enrichment process"""
|
||||||
|
|
||||||
# Mock the calendar submission to avoid network requests
|
# Mock the calendar submission to avoid network requests
|
||||||
mock_calendar = mocker.patch.object(RemoteCalendar, 'submit')
|
mock_calendar = mocker.patch.object(RemoteCalendar, "submit")
|
||||||
|
|
||||||
# Create a function that returns a new timestamp for each call
|
# Create a function that returns a new timestamp for each call
|
||||||
def side_effect(digest):
|
def side_effect(digest):
|
||||||
|
@ -197,19 +204,30 @@ def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
|
||||||
assert timestamp_media.get("verified") == True
|
assert timestamp_media.get("verified") == True
|
||||||
assert timestamp_media.get("attestation_count") == 1
|
assert timestamp_media.get("attestation_count") == 1
|
||||||
|
|
||||||
def test_full_enriching_one_calendar_error(setup_module, sample_file_path, sample_media, mocker, pending_timestamp_file):
|
|
||||||
|
def test_full_enriching_one_calendar_error(
|
||||||
|
setup_module, sample_file_path, sample_media, mocker, pending_timestamp_file
|
||||||
|
):
|
||||||
"""Test enrichment when one calendar server returns an error"""
|
"""Test enrichment when one calendar server returns an error"""
|
||||||
# Mock the calendar submission to raise an exception
|
# Mock the calendar submission to raise an exception
|
||||||
mock_calendar = mocker.patch.object(RemoteCalendar, 'submit')
|
mock_calendar = mocker.patch.object(RemoteCalendar, "submit")
|
||||||
|
|
||||||
test_timestamp = Timestamp(bytes.fromhex("583988e03646c26fa290c5c2408540a2f4e2aa9be087aa4546aefb531385b935"))
|
test_timestamp = Timestamp(bytes.fromhex("583988e03646c26fa290c5c2408540a2f4e2aa9be087aa4546aefb531385b935"))
|
||||||
# Add a bitcoin attestation to the test timestamp
|
# Add a bitcoin attestation to the test timestamp
|
||||||
bitcoin = BitcoinBlockHeaderAttestation(783000)
|
bitcoin = BitcoinBlockHeaderAttestation(783000)
|
||||||
test_timestamp.attestations.add(bitcoin)
|
test_timestamp.attestations.add(bitcoin)
|
||||||
|
|
||||||
mock_calendar.side_effect = [test_timestamp, Exception("Calendar server error")]
|
mock_calendar.side_effect = [test_timestamp, Exception("Calendar server error")]
|
||||||
|
|
||||||
ots = setup_module("opentimestamps_enricher", {"calendar_urls": ["https://alice.btc.calendar.opentimestamps.org", "https://bob.btc.calendar.opentimestamps.org"]})
|
ots = setup_module(
|
||||||
|
"opentimestamps_enricher",
|
||||||
|
{
|
||||||
|
"calendar_urls": [
|
||||||
|
"https://alice.btc.calendar.opentimestamps.org",
|
||||||
|
"https://bob.btc.calendar.opentimestamps.org",
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
# Create test metadata with sample file
|
# Create test metadata with sample file
|
||||||
metadata = Metadata().set_url("https://example.com")
|
metadata = Metadata().set_url("https://example.com")
|
||||||
|
@ -221,15 +239,15 @@ def test_full_enriching_one_calendar_error(setup_module, sample_file_path, sampl
|
||||||
|
|
||||||
# Verify results
|
# Verify results
|
||||||
assert metadata.get("opentimestamped") == True
|
assert metadata.get("opentimestamped") == True
|
||||||
assert metadata.get("opentimestamps_count") == 1 # only alice worked, not bob
|
assert metadata.get("opentimestamps_count") == 1 # only alice worked, not bob
|
||||||
|
|
||||||
|
|
||||||
def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_media, mocker):
|
def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_media, mocker):
|
||||||
"""Test enrichment when calendar servers return errors"""
|
"""Test enrichment when calendar servers return errors"""
|
||||||
# Mock the calendar submission to raise an exception
|
# Mock the calendar submission to raise an exception
|
||||||
mock_calendar = mocker.patch.object(RemoteCalendar, 'submit')
|
mock_calendar = mocker.patch.object(RemoteCalendar, "submit")
|
||||||
mock_calendar.side_effect = Exception("Calendar server error")
|
mock_calendar.side_effect = Exception("Calendar server error")
|
||||||
|
|
||||||
|
|
||||||
ots = setup_module("opentimestamps_enricher")
|
ots = setup_module("opentimestamps_enricher")
|
||||||
|
|
||||||
# Create test metadata with sample file
|
# Create test metadata with sample file
|
||||||
|
@ -244,6 +262,7 @@ def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_me
|
||||||
assert metadata.get("opentimestamped") == False
|
assert metadata.get("opentimestamped") == False
|
||||||
assert metadata.get("opentimestamps_count") is None
|
assert metadata.get("opentimestamps_count") is None
|
||||||
|
|
||||||
|
|
||||||
def test_no_files_to_stamp(setup_module):
|
def test_no_files_to_stamp(setup_module):
|
||||||
"""Test enrichment with no files to timestamp"""
|
"""Test enrichment with no files to timestamp"""
|
||||||
ots = setup_module("opentimestamps_enricher")
|
ots = setup_module("opentimestamps_enricher")
|
||||||
|
|
Ładowanie…
Reference in New Issue