mirror
/
auto-archiver
kopia lustrzana https://github.com/bellingcat/auto-archiver
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Tweaks to settings page + more declarations in manifests

pull/217/head
Patrick Robertson 2025-02-27 15:21:11 +00:00
rodzic efe9fdf915
commit 1e92c03b1d
6 zmienionych plików z 91 dodań i 38 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
scripts/generate_settings_page.py
Wyświetl plik

@ -31,7 +31,7 @@ output_schame = {
'configs': module.configs or None 'configs': module.configs or None
} }
) for module in all_modules_ordered_by_type), ) for module in all_modules_ordered_by_type),
'steps': dict((module_type, [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES), 'steps': dict((f"{module_type}s", [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES),
'configs': [m.name for m in all_modules_ordered_by_type if m.configs], 'configs': [m.name for m in all_modules_ordered_by_type if m.configs],
'module_types': MODULE_TYPES, 'module_types': MODULE_TYPES,
} }

52
scripts/settings/src/App.tsx
Wyświetl plik

@ -81,7 +81,12 @@ function FileDrop({ setYamlFile }) {
function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues }: { stepType: string, setEnabledModules: any, enabledModules: any, configValues: any }) { function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues }: { stepType: string, setEnabledModules: any, enabledModules: any, configValues: any }) {
const [showError, setShowError] = useState(false); const [showError, setShowError] = useState(false);
const [activeId, setActiveId] = useState(null); const [activeId, setActiveId] = useState(null);
const [items, setItems] = useState<string[]>(enabledModules[stepType].map(([name, enabled]: [string, boolean]) => name)); const [items, setItems] = useState<string[]>([]);
useEffect(() => {
setItems(enabledModules[stepType].map(([name, enabled]: [string, boolean]) => name));
}
, [enabledModules]);
const toggleModule = (event: any) => { const toggleModule = (event: any) => {
// make sure that 'feeder' and 'formatter' types only have one value // make sure that 'feeder' and 'formatter' types only have one value
@ -100,11 +105,12 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues
} else { } else {
setShowError(false); setShowError(false);
} }
let newEnabledModules = { ...enabledModules }; let newEnabledModules = Object.fromEntries(Object.keys(enabledModules).map((type : string) => {
newEnabledModules[stepType] = enabledModules[stepType].map(([m, enabled]: [string, boolean]) => { return [type, enabledModules[type].map(([m, enabled]: [string, boolean]) => {
return (m === name) ? [m, checked] : [m, enabled]; return (m === name) ? [m, checked] : [m, enabled];
} })];
); }
));
setEnabledModules(newEnabledModules); setEnabledModules(newEnabledModules);
} }
@ -134,17 +140,16 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues
return newArray.indexOf(a[0]) - newArray.indexOf(b[0]); return newArray.indexOf(a[0]) - newArray.indexOf(b[0]);
}) })
setEnabledModules(newEnabledModules); setEnabledModules(newEnabledModules);
setItems(newArray);
} }
}; };
return ( return (
<> <>
<Box sx={{ my: 4 }}> <Box sx={{ my: 4 }}>
<Typography id={stepType} variant="h6" style={{ textTransform: 'capitalize' }} > <Typography id={stepType} variant="h6" style={{ textTransform: 'capitalize' }} >
{stepType}s {stepType}
</Typography> </Typography>
<Typography variant="body1" > <Typography variant="body1" >
Select the {stepType}s you wish to enable. You can drag and drop them to reorder them. Select the {stepType} you wish to enable. You can drag and drop them to reorder them.
</Typography> </Typography>
</Box> </Box>
{showError ? <Typography variant="body1" color="error" >Only one {stepType} can be enabled at a time.</Typography> : null} {showError ? <Typography variant="body1" color="error" >Only one {stepType} can be enabled at a time.</Typography> : null}
@ -186,7 +191,7 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues
export default function App() { export default function App() {
const [yamlFile, setYamlFile] = useState<Document>(new Document()); const [yamlFile, setYamlFile] = useState<Document>(new Document());
const [enabledModules, setEnabledModules] = useState<{}>(Object.fromEntries(module_types.map(type => [type, steps[type].map((name: string) => [name, false])]))); const [enabledModules, setEnabledModules] = useState<{}>(Object.fromEntries(Object.keys(steps).map(type => [type, steps[type].map((name: string) => [name, false])])));
const [configValues, setConfigValues] = useState<{}>( const [configValues, setConfigValues] = useState<{}>(
Object.keys(modules).reduce((acc, module) => { Object.keys(modules).reduce((acc, module) => {
acc[module] = {}; acc[module] = {};
@ -202,7 +207,7 @@ export default function App() {
// create a yaml file from // create a yaml file from
const finalYaml = { const finalYaml = {
'steps': Object.keys(stepsConfig).reduce((acc, stepType) => { 'steps': Object.keys(steps).reduce((acc, stepType) => {
acc[stepType] = stepsConfig[stepType].filter(([name, enabled]: [string, boolean]) => enabled).map(([name, enabled]: [string, boolean]) => name); acc[stepType] = stepsConfig[stepType].filter(([name, enabled]: [string, boolean]) => enabled).map(([name, enabled]: [string, boolean]) => name);
return acc; return acc;
}, {}) }, {})
@ -257,10 +262,27 @@ export default function App() {
let settings = yamlFile.toJS(); let settings = yamlFile.toJS();
// make a deep copy of settings // make a deep copy of settings
let newEnabledModules = Object.keys(settings['steps']).map((step: string) => { let stepSettings = settings['steps'];
return settings['steps'][step]; let newEnabledModules = Object.fromEntries(Object.keys(steps).map((type: string) => {
}).flat(); return [type, steps[type].map((name: string) => {
newEnabledModules = newEnabledModules.filter((m: string, i: number) => newEnabledModules.indexOf(m) === i); return [name, stepSettings[type].indexOf(name) !== -1];
}).sort((a, b) => {
let aIndex = stepSettings[type].indexOf(a[0]);
let bIndex = stepSettings[type].indexOf(b[0]);
if (aIndex === -1 && bIndex === -1) {
return a - b;
}
if (bIndex === -1) {
return -1;
}
if (aIndex === -1) {
return 1;
}
return aIndex - bIndex;
})];
}).sort((a, b) => {
return module_types.indexOf(a[0]) - module_types.indexOf(b[0]);
}));
setEnabledModules(newEnabledModules); setEnabledModules(newEnabledModules);
}, [yamlFile]); }, [yamlFile]);

20
scripts/settings/src/StepCard.tsx
Wyświetl plik

@ -20,7 +20,7 @@ import {
MenuItem, MenuItem,
FormControl, FormControl,
FormControlLabel, FormControlLabel,
InputLabel, Textarea,
FormHelperText, FormHelperText,
TextField, TextField,
Stack, Stack,
@ -141,13 +141,15 @@ function ConfigPanel({ module, open, setOpen, configValues }: { module: any, ope
const value = configValues[module.name][config_value] || config_args.default; const value = configValues[module.name][config_value] || config_args.default;
return ( return (
<Box key={config_value}> <Box key={config_value}>
<Typography variant='body1'>{config_display_name}</Typography> <Typography variant='body1' style={{fontWeight : 'bold'}}>{config_display_name}</Typography>
<FormControl size="small"> <FormControl size="small">
{ config_args.type === 'bool' ? { config_args.type === 'bool' ?
<FormControlLabel control={
<Checkbox defaultChecked={value} size="small" id={`${module}.${config_value}`} <Checkbox defaultChecked={value} size="small" id={`${module}.${config_value}`}
onChange={(e) => { onChange={(e) => {
setConfigValue(config_value, e.target.checked); setConfigValue(config_value, e.target.checked);
}} }}
/>} label={config_args.help}
/> />
: :
( (
@ -167,16 +169,16 @@ function ConfigPanel({ module, open, setOpen, configValues }: { module: any, ope
</Select> </Select>
: :
( config_args.type === 'json_loader' ? ( config_args.type === 'json_loader' ?
<TextField size="small" id={`${module}.${config_value}`} defaultValue={JSON.stringify(value)} onChange={ <TextField multiline size="small" id={`${module}.${config_value}`} defaultValue={JSON.stringify(value, null, 2)} rows={6} onChange={
(e) => { (e) => {
try { try {
val = JSON.parse(e.target.value); let val = JSON.parse(e.target.value);
setConfigValue(config_value, val); setConfigValue(config_value, val);
} catch (e) { } catch (e) {
console.log(e); console.log(e);
} }
} }
} type='text' /> } />
: :
<TextField size="small" id={`${module}.${config_value}`} defaultValue={value} type={config_args.type === 'int' ? 'number' : 'text'} <TextField size="small" id={`${module}.${config_value}`} defaultValue={value} type={config_args.type === 'int' ? 'number' : 'text'}
onChange={(e) => { onChange={(e) => {
@ -185,8 +187,10 @@ function ConfigPanel({ module, open, setOpen, configValues }: { module: any, ope
) )
) )
} }
<FormHelperText style={{ textTransform: 'capitalize'}}>{config_args.help}</FormHelperText> {config_args.type !== 'bool' && (
</FormControl> <FormHelperText >{config_args.help}</FormHelperText>
)}
</FormControl>
</Box> </Box>
); );
})} })}

30
scripts/settings/src/schema.json
Wyświetl plik

@ -799,6 +799,7 @@
"configs": { "configs": {
"timeout": { "timeout": {
"default": 15, "default": 15,
"type": "int",
"help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually." "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually."
}, },
"if_not_archived_within": { "if_not_archived_within": {
@ -826,6 +827,7 @@
"configs": { "configs": {
"timeout": { "timeout": {
"default": 15, "default": 15,
"type": "int",
"help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually." "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually."
}, },
"if_not_archived_within": { "if_not_archived_within": {
@ -1046,18 +1048,22 @@
"configs": { "configs": {
"width": { "width": {
"default": 1280, "default": 1280,
"type": "int",
"help": "width of the screenshots" "help": "width of the screenshots"
}, },
"height": { "height": {
"default": 720, "default": 720,
"type": "int",
"help": "height of the screenshots" "help": "height of the screenshots"
}, },
"timeout": { "timeout": {
"default": 60, "default": 60,
"type": "int",
"help": "timeout for taking the screenshot" "help": "timeout for taking the screenshot"
}, },
"sleep_before_screenshot": { "sleep_before_screenshot": {
"default": 4, "default": 4,
"type": "int",
"help": "seconds to wait for the pages to load before taking screenshot" "help": "seconds to wait for the pages to load before taking screenshot"
}, },
"http_proxy": { "http_proxy": {
@ -1066,29 +1072,35 @@
}, },
"save_to_pdf": { "save_to_pdf": {
"default": false, "default": false,
"type": "bool",
"help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter" "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"
}, },
"print_options": { "print_options": {
"default": {}, "default": {},
"help": "options to pass to the pdf printer" "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information",
"type": "json_loader"
} }
} }
}, },
"configs": { "configs": {
"width": { "width": {
"default": 1280, "default": 1280,
"type": "int",
"help": "width of the screenshots" "help": "width of the screenshots"
}, },
"height": { "height": {
"default": 720, "default": 720,
"type": "int",
"help": "height of the screenshots" "help": "height of the screenshots"
}, },
"timeout": { "timeout": {
"default": 60, "default": 60,
"type": "int",
"help": "timeout for taking the screenshot" "help": "timeout for taking the screenshot"
}, },
"sleep_before_screenshot": { "sleep_before_screenshot": {
"default": 4, "default": 4,
"type": "int",
"help": "seconds to wait for the pages to load before taking screenshot" "help": "seconds to wait for the pages to load before taking screenshot"
}, },
"http_proxy": { "http_proxy": {
@ -1097,11 +1109,13 @@
}, },
"save_to_pdf": { "save_to_pdf": {
"default": false, "default": false,
"type": "bool",
"help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter" "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"
}, },
"print_options": { "print_options": {
"default": {}, "default": {},
"help": "options to pass to the pdf printer" "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information",
"type": "json_loader"
} }
} }
}, },
@ -2007,13 +2021,13 @@
} }
}, },
"steps": { "steps": {
"feeder": [ "feeders": [
"cli_feeder", "cli_feeder",
"gsheet_feeder", "gsheet_feeder",
"atlos_feeder", "atlos_feeder",
"csv_feeder" "csv_feeder"
], ],
"extractor": [ "extractors": [
"wayback_extractor_enricher", "wayback_extractor_enricher",
"wacz_extractor_enricher", "wacz_extractor_enricher",
"instagram_api_extractor", "instagram_api_extractor",
@ -2025,7 +2039,7 @@
"vk_extractor", "vk_extractor",
"telegram_extractor" "telegram_extractor"
], ],
"enricher": [ "enrichers": [
"wayback_extractor_enricher", "wayback_extractor_enricher",
"wacz_extractor_enricher", "wacz_extractor_enricher",
"metadata_enricher", "metadata_enricher",
@ -2038,20 +2052,20 @@
"ssl_enricher", "ssl_enricher",
"hash_enricher" "hash_enricher"
], ],
"database": [ "databases": [
"console_db", "console_db",
"atlos_db", "atlos_db",
"api_db", "api_db",
"csv_db", "csv_db",
"gsheet_db" "gsheet_db"
], ],
"storage": [ "storages": [
"local_storage", "local_storage",
"gdrive_storage", "gdrive_storage",
"atlos_storage", "atlos_storage",
"s3_storage" "s3_storage"
], ],
"formatter": [ "formatters": [
"html_formatter", "html_formatter",
"mute_formatter" "mute_formatter"
] ]

24
src/auto_archiver/modules/screenshot_enricher/__manifest__.py
Wyświetl plik

@ -6,13 +6,25 @@
"python": ["loguru", "selenium"], "python": ["loguru", "selenium"],
}, },
"configs": { "configs": {
"width": {"default": 1280, "help": "width of the screenshots"}, "width": {"default": 1280,
"height": {"default": 720, "help": "height of the screenshots"}, "type": "int",
"timeout": {"default": 60, "help": "timeout for taking the screenshot"}, "help": "width of the screenshots"},
"sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"}, "height": {"default": 720,
"type": "int",
"help": "height of the screenshots"},
"timeout": {"default": 60,
"type": "int",
"help": "timeout for taking the screenshot"},
"sleep_before_screenshot": {"default": 4,
"type": "int",
"help": "seconds to wait for the pages to load before taking screenshot"},
"http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"}, "http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"},
"save_to_pdf": {"default": False, "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"}, "save_to_pdf": {"default": False,
"print_options": {"default": {}, "help": "options to pass to the pdf printer"} "type": "bool",
"help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"},
"print_options": {"default": {},
"help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information",
"type": "json_loader"},
}, },
"description": """ "description": """
Captures screenshots and optionally saves web pages as PDFs using a WebDriver. Captures screenshots and optionally saves web pages as PDFs using a WebDriver.

1
src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py
Wyświetl plik

@ -9,6 +9,7 @@
"configs": { "configs": {
"timeout": { "timeout": {
"default": 15, "default": 15,
"type": "int",
"help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually.", "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually.",
}, },
"if_not_archived_within": { "if_not_archived_within": {