mirror
/
auto-archiver
kopia lustrzana https://github.com/bellingcat/auto-archiver
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Tweaks to settings page + more declarations in manifests

pull/217/head
Patrick Robertson 2025-02-27 15:21:11 +00:00
rodzic efe9fdf915
commit 1e92c03b1d
6 zmienionych plików z 91 dodań i 38 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
scripts/generate_settings_page.py
Wyświetl plik

@ -31,7 +31,7 @@ output_schame = {
'configs': module.configs or None
}
) for module in all_modules_ordered_by_type),
'steps': dict((module_type, [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES),
'steps': dict((f"{module_type}s", [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES),
'configs': [m.name for m in all_modules_ordered_by_type if m.configs],
'module_types': MODULE_TYPES,
}

52
scripts/settings/src/App.tsx
Wyświetl plik

@ -81,7 +81,12 @@ function FileDrop({ setYamlFile }) {
function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues }: { stepType: string, setEnabledModules: any, enabledModules: any, configValues: any }) {
const [showError, setShowError] = useState(false);
const [activeId, setActiveId] = useState(null);
const [items, setItems] = useState<string[]>(enabledModules[stepType].map(([name, enabled]: [string, boolean]) => name));
const [items, setItems] = useState<string[]>([]);
useEffect(() => {
setItems(enabledModules[stepType].map(([name, enabled]: [string, boolean]) => name));
}
, [enabledModules]);
const toggleModule = (event: any) => {
// make sure that 'feeder' and 'formatter' types only have one value
@ -100,11 +105,12 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues
} else {
setShowError(false);
}
let newEnabledModules = { ...enabledModules };
newEnabledModules[stepType] = enabledModules[stepType].map(([m, enabled]: [string, boolean]) => {
return (m === name) ? [m, checked] : [m, enabled];
}
);
let newEnabledModules = Object.fromEntries(Object.keys(enabledModules).map((type : string) => {
return [type, enabledModules[type].map(([m, enabled]: [string, boolean]) => {
return (m === name) ? [m, checked] : [m, enabled];
})];
}
));
setEnabledModules(newEnabledModules);
}
@ -134,17 +140,16 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues
return newArray.indexOf(a[0]) - newArray.indexOf(b[0]);
})
setEnabledModules(newEnabledModules);
setItems(newArray);
}
};
return (
<>
<Box sx={{ my: 4 }}>
<Typography id={stepType} variant="h6" style={{ textTransform: 'capitalize' }} >
{stepType}s
{stepType}
</Typography>
<Typography variant="body1" >
Select the {stepType}s you wish to enable. You can drag and drop them to reorder them.
Select the {stepType} you wish to enable. You can drag and drop them to reorder them.
</Typography>
</Box>
{showError ? <Typography variant="body1" color="error" >Only one {stepType} can be enabled at a time.</Typography> : null}
@ -186,7 +191,7 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues
export default function App() {
const [yamlFile, setYamlFile] = useState<Document>(new Document());
const [enabledModules, setEnabledModules] = useState<{}>(Object.fromEntries(module_types.map(type => [type, steps[type].map((name: string) => [name, false])])));
const [enabledModules, setEnabledModules] = useState<{}>(Object.fromEntries(Object.keys(steps).map(type => [type, steps[type].map((name: string) => [name, false])])));
const [configValues, setConfigValues] = useState<{}>(
Object.keys(modules).reduce((acc, module) => {
acc[module] = {};
@ -202,7 +207,7 @@ export default function App() {
// create a yaml file from
const finalYaml = {
'steps': Object.keys(stepsConfig).reduce((acc, stepType) => {
'steps': Object.keys(steps).reduce((acc, stepType) => {
acc[stepType] = stepsConfig[stepType].filter(([name, enabled]: [string, boolean]) => enabled).map(([name, enabled]: [string, boolean]) => name);
return acc;
}, {})
@ -257,10 +262,27 @@ export default function App() {
let settings = yamlFile.toJS();
// make a deep copy of settings
let newEnabledModules = Object.keys(settings['steps']).map((step: string) => {
return settings['steps'][step];
}).flat();
newEnabledModules = newEnabledModules.filter((m: string, i: number) => newEnabledModules.indexOf(m) === i);
let stepSettings = settings['steps'];
let newEnabledModules = Object.fromEntries(Object.keys(steps).map((type: string) => {
return [type, steps[type].map((name: string) => {
return [name, stepSettings[type].indexOf(name) !== -1];
}).sort((a, b) => {
let aIndex = stepSettings[type].indexOf(a[0]);
let bIndex = stepSettings[type].indexOf(b[0]);
if (aIndex === -1 && bIndex === -1) {
return a - b;
}
if (bIndex === -1) {
return -1;
}
if (aIndex === -1) {
return 1;
}
return aIndex - bIndex;
})];
}).sort((a, b) => {
return module_types.indexOf(a[0]) - module_types.indexOf(b[0]);
}));
setEnabledModules(newEnabledModules);
}, [yamlFile]);

20
scripts/settings/src/StepCard.tsx
Wyświetl plik

@ -20,7 +20,7 @@ import {
MenuItem,
FormControl,
FormControlLabel,
InputLabel,
Textarea,
FormHelperText,
TextField,
Stack,
@ -141,13 +141,15 @@ function ConfigPanel({ module, open, setOpen, configValues }: { module: any, ope
const value = configValues[module.name][config_value] || config_args.default;
return (
<Box key={config_value}>
<Typography variant='body1'>{config_display_name}</Typography>
<Typography variant='body1' style={{fontWeight : 'bold'}}>{config_display_name}</Typography>
<FormControl size="small">
{ config_args.type === 'bool' ?
{ config_args.type === 'bool' ?
<FormControlLabel control={
<Checkbox defaultChecked={value} size="small" id={`${module}.${config_value}`}
onChange={(e) => {
setConfigValue(config_value, e.target.checked);
}}
/>} label={config_args.help}
/>
:
(
@ -167,16 +169,16 @@ function ConfigPanel({ module, open, setOpen, configValues }: { module: any, ope
</Select>
:
( config_args.type === 'json_loader' ?
<TextField size="small" id={`${module}.${config_value}`} defaultValue={JSON.stringify(value)} onChange={
<TextField multiline size="small" id={`${module}.${config_value}`} defaultValue={JSON.stringify(value, null, 2)} rows={6} onChange={
(e) => {
try {
val = JSON.parse(e.target.value);
let val = JSON.parse(e.target.value);
setConfigValue(config_value, val);
} catch (e) {
console.log(e);
}
}
} type='text' />
} />
:
<TextField size="small" id={`${module}.${config_value}`} defaultValue={value} type={config_args.type === 'int' ? 'number' : 'text'}
onChange={(e) => {
@ -185,8 +187,10 @@ function ConfigPanel({ module, open, setOpen, configValues }: { module: any, ope
)
)
}
<FormHelperText style={{ textTransform: 'capitalize'}}>{config_args.help}</FormHelperText>
</FormControl>
{config_args.type !== 'bool' && (
<FormHelperText >{config_args.help}</FormHelperText>
)}
</FormControl>
</Box>
);
})}

30
scripts/settings/src/schema.json
Wyświetl plik

@ -799,6 +799,7 @@
"configs": {
"timeout": {
"default": 15,
"type": "int",
"help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually."
},
"if_not_archived_within": {
@ -826,6 +827,7 @@
"configs": {
"timeout": {
"default": 15,
"type": "int",
"help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually."
},
"if_not_archived_within": {
@ -1046,18 +1048,22 @@
"configs": {
"width": {
"default": 1280,
"type": "int",
"help": "width of the screenshots"
},
"height": {
"default": 720,
"type": "int",
"help": "height of the screenshots"
},
"timeout": {
"default": 60,
"type": "int",
"help": "timeout for taking the screenshot"
},
"sleep_before_screenshot": {
"default": 4,
"type": "int",
"help": "seconds to wait for the pages to load before taking screenshot"
},
"http_proxy": {
@ -1066,29 +1072,35 @@
},
"save_to_pdf": {
"default": false,
"type": "bool",
"help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"
},
"print_options": {
"default": {},
"help": "options to pass to the pdf printer"
"help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information",
"type": "json_loader"
}
}
},
"configs": {
"width": {
"default": 1280,
"type": "int",
"help": "width of the screenshots"
},
"height": {
"default": 720,
"type": "int",
"help": "height of the screenshots"
},
"timeout": {
"default": 60,
"type": "int",
"help": "timeout for taking the screenshot"
},
"sleep_before_screenshot": {
"default": 4,
"type": "int",
"help": "seconds to wait for the pages to load before taking screenshot"
},
"http_proxy": {
@ -1097,11 +1109,13 @@
},
"save_to_pdf": {
"default": false,
"type": "bool",
"help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"
},
"print_options": {
"default": {},
"help": "options to pass to the pdf printer"
"help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information",
"type": "json_loader"
}
}
},
@ -2007,13 +2021,13 @@
}
},
"steps": {
"feeder": [
"feeders": [
"cli_feeder",
"gsheet_feeder",
"atlos_feeder",
"csv_feeder"
],
"extractor": [
"extractors": [
"wayback_extractor_enricher",
"wacz_extractor_enricher",
"instagram_api_extractor",
@ -2025,7 +2039,7 @@
"vk_extractor",
"telegram_extractor"
],
"enricher": [
"enrichers": [
"wayback_extractor_enricher",
"wacz_extractor_enricher",
"metadata_enricher",
@ -2038,20 +2052,20 @@
"ssl_enricher",
"hash_enricher"
],
"database": [
"databases": [
"console_db",
"atlos_db",
"api_db",
"csv_db",
"gsheet_db"
],
"storage": [
"storages": [
"local_storage",
"gdrive_storage",
"atlos_storage",
"s3_storage"
],
"formatter": [
"formatters": [
"html_formatter",
"mute_formatter"
]

24
src/auto_archiver/modules/screenshot_enricher/__manifest__.py
Wyświetl plik

@ -6,13 +6,25 @@
"python": ["loguru", "selenium"],
},
"configs": {
"width": {"default": 1280, "help": "width of the screenshots"},
"height": {"default": 720, "help": "height of the screenshots"},
"timeout": {"default": 60, "help": "timeout for taking the screenshot"},
"sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"},
"width": {"default": 1280,
"type": "int",
"help": "width of the screenshots"},
"height": {"default": 720,
"type": "int",
"help": "height of the screenshots"},
"timeout": {"default": 60,
"type": "int",
"help": "timeout for taking the screenshot"},
"sleep_before_screenshot": {"default": 4,
"type": "int",
"help": "seconds to wait for the pages to load before taking screenshot"},
"http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"},
"save_to_pdf": {"default": False, "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"},
"print_options": {"default": {}, "help": "options to pass to the pdf printer"}
"save_to_pdf": {"default": False,
"type": "bool",
"help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"},
"print_options": {"default": {},
"help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information",
"type": "json_loader"},
},
"description": """
Captures screenshots and optionally saves web pages as PDFs using a WebDriver.

1
src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py
Wyświetl plik

@ -9,6 +9,7 @@
"configs": {
"timeout": {
"default": 15,
"type": "int",
"help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually.",
},
"if_not_archived_within": {