kopia lustrzana https://github.com/bellingcat/auto-archiver
Add configs for path to write and read wacz archives (#93)
Co-authored-by: msramalho <19508417+msramalho@users.noreply.github.com>pull/102/head
rodzic
7a2119e6e9
commit
f7839a99cc
|
@ -28,4 +28,4 @@ COPY ./src/ .
|
||||||
ENTRYPOINT ["pipenv", "run", "python3", "-m", "auto_archiver"]
|
ENTRYPOINT ["pipenv", "run", "python3", "-m", "auto_archiver"]
|
||||||
|
|
||||||
# should be executed with 2 volumes (3 if local_storage is used)
|
# should be executed with 2 volumes (3 if local_storage is used)
|
||||||
# docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive aa pipenv run python3 -m auto_archiver --config secrets/orchestration.yaml
|
# docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive aa pipenv run python3 -m auto_archiver --config secrets/orchestration.yaml
|
||||||
|
|
|
@ -28,7 +28,6 @@ class WaczArchiverEnricher(Enricher, Archiver):
|
||||||
return {
|
return {
|
||||||
"profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."},
|
"profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."},
|
||||||
"docker_commands": {"default": None, "help":"if a custom docker invocation is needed"},
|
"docker_commands": {"default": None, "help":"if a custom docker invocation is needed"},
|
||||||
"browsertrix_home": {"default": None, "help": "Path to use with the custom browsertrix file locations, useful together with docker_commands"},
|
|
||||||
"timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"},
|
"timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"},
|
||||||
"extract_media": {"default": True, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media. The .wacz file will be kept untouched."}
|
"extract_media": {"default": True, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media. The .wacz file will be kept untouched."}
|
||||||
}
|
}
|
||||||
|
@ -48,50 +47,45 @@ class WaczArchiverEnricher(Enricher, Archiver):
|
||||||
url = to_enrich.get_url()
|
url = to_enrich.get_url()
|
||||||
|
|
||||||
collection = str(uuid.uuid4())[0:8]
|
collection = str(uuid.uuid4())[0:8]
|
||||||
browsertrix_home = self.browsertrix_home or os.path.abspath(ArchivingContext.get_tmp_dir())
|
browsertrix_home_host = os.environ.get('BROWSERTRIX_HOME_HOST') or os.path.abspath(ArchivingContext.get_tmp_dir())
|
||||||
|
browsertrix_home_container = os.environ.get('BROWSERTRIX_HOME_CONTAINER') or browsertrix_home_host
|
||||||
|
|
||||||
if os.environ.get('RUNNING_IN_DOCKER', 0) == '1':
|
cmd = [
|
||||||
logger.debug(f"generating WACZ without Docker for {url=}")
|
"crawl",
|
||||||
|
"--url", url,
|
||||||
|
"--scopeType", "page",
|
||||||
|
"--generateWACZ",
|
||||||
|
"--text",
|
||||||
|
"--screenshot", "fullPage",
|
||||||
|
"--collection", collection,
|
||||||
|
"--id", collection,
|
||||||
|
"--saveState", "never",
|
||||||
|
"--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
|
||||||
|
"--behaviorTimeout", str(self.timeout),
|
||||||
|
"--timeout", str(self.timeout)]
|
||||||
|
|
||||||
cmd = [
|
# call docker if explicitly enabled or we are running on the host (not in docker)
|
||||||
"crawl",
|
use_docker = os.environ.get('WACZ_ENABLE_DOCKER') or not os.environ.get('RUNNING_IN_DOCKER')
|
||||||
"--url", url,
|
|
||||||
"--scopeType", "page",
|
|
||||||
"--generateWACZ",
|
|
||||||
"--text",
|
|
||||||
"--screenshot", "fullPage",
|
|
||||||
"--collection", collection,
|
|
||||||
"--id", collection,
|
|
||||||
"--saveState", "never",
|
|
||||||
"--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
|
|
||||||
"--behaviorTimeout", str(self.timeout),
|
|
||||||
"--timeout", str(self.timeout)]
|
|
||||||
|
|
||||||
if self.profile:
|
if use_docker:
|
||||||
cmd.extend(["--profile", os.path.join("/app", str(self.profile))])
|
|
||||||
else:
|
|
||||||
logger.debug(f"generating WACZ in Docker for {url=}")
|
logger.debug(f"generating WACZ in Docker for {url=}")
|
||||||
|
logger.debug(f"{browsertrix_home_host=} {browsertrix_home_container=}")
|
||||||
if not self.docker_commands:
|
if not self.docker_commands:
|
||||||
self.docker_commands = ["docker", "run", "--rm", "-v", f"{browsertrix_home}:/crawls/", "webrecorder/browsertrix-crawler"]
|
self.docker_commands = ["docker", "run", "--rm", "-v", f"{browsertrix_home_host}:/crawls/", "webrecorder/browsertrix-crawler"]
|
||||||
cmd = self.docker_commands + [
|
cmd = self.docker_commands + cmd
|
||||||
"crawl",
|
|
||||||
"--url", url,
|
|
||||||
"--scopeType", "page",
|
|
||||||
"--generateWACZ",
|
|
||||||
"--text",
|
|
||||||
"--screenshot", "fullPage",
|
|
||||||
"--collection", collection,
|
|
||||||
"--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
|
|
||||||
"--behaviorTimeout", str(self.timeout),
|
|
||||||
"--timeout", str(self.timeout)
|
|
||||||
]
|
|
||||||
|
|
||||||
if self.profile:
|
if self.profile:
|
||||||
profile_fn = os.path.join(browsertrix_home, "profile.tar.gz")
|
profile_fn = os.path.join(browsertrix_home_container, "profile.tar.gz")
|
||||||
logger.debug(f"copying {self.profile} to {profile_fn}")
|
logger.debug(f"copying {self.profile} to {profile_fn}")
|
||||||
shutil.copyfile(self.profile, profile_fn)
|
shutil.copyfile(self.profile, profile_fn)
|
||||||
cmd.extend(["--profile", os.path.join("/crawls", "profile.tar.gz")])
|
cmd.extend(["--profile", os.path.join("/crawls", "profile.tar.gz")])
|
||||||
|
|
||||||
|
else:
|
||||||
|
logger.debug(f"generating WACZ without Docker for {url=}")
|
||||||
|
|
||||||
|
if self.profile:
|
||||||
|
cmd.extend(["--profile", os.path.join("/app", str(self.profile))])
|
||||||
|
|
||||||
try:
|
try:
|
||||||
logger.info(f"Running browsertrix-crawler: {' '.join(cmd)}")
|
logger.info(f"Running browsertrix-crawler: {' '.join(cmd)}")
|
||||||
subprocess.run(cmd, check=True)
|
subprocess.run(cmd, check=True)
|
||||||
|
@ -99,10 +93,10 @@ class WaczArchiverEnricher(Enricher, Archiver):
|
||||||
logger.error(f"WACZ generation failed: {e}")
|
logger.error(f"WACZ generation failed: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if os.getenv('RUNNING_IN_DOCKER'):
|
if use_docker:
|
||||||
filename = os.path.join("collections", collection, f"{collection}.wacz")
|
filename = os.path.join(browsertrix_home_container, "collections", collection, f"{collection}.wacz")
|
||||||
else:
|
else:
|
||||||
filename = os.path.join(browsertrix_home, "collections", collection, f"{collection}.wacz")
|
filename = os.path.join("collections", collection, f"{collection}.wacz")
|
||||||
|
|
||||||
if not os.path.exists(filename):
|
if not os.path.exists(filename):
|
||||||
logger.warning(f"Unable to locate and upload WACZ {filename=}")
|
logger.warning(f"Unable to locate and upload WACZ {filename=}")
|
||||||
|
|
Ładowanie…
Reference in New Issue