From 1e92c03b1d0216e4522f4072d1815a59f8d8f825 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Thu, 27 Feb 2025 15:21:11 +0000 Subject: [PATCH] Tweaks to settings page + more declarations in manifests --- scripts/generate_settings_page.py | 2 +- scripts/settings/src/App.tsx | 52 +++++++++++++------ scripts/settings/src/StepCard.tsx | 20 ++++--- scripts/settings/src/schema.json | 30 ++++++++--- .../screenshot_enricher/__manifest__.py | 24 ++++++--- .../__manifest__.py | 1 + 6 files changed, 91 insertions(+), 38 deletions(-) diff --git a/scripts/generate_settings_page.py b/scripts/generate_settings_page.py index 632d782..cb3d452 100644 --- a/scripts/generate_settings_page.py +++ b/scripts/generate_settings_page.py @@ -31,7 +31,7 @@ output_schame = { 'configs': module.configs or None } ) for module in all_modules_ordered_by_type), - 'steps': dict((module_type, [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES), + 'steps': dict((f"{module_type}s", [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES), 'configs': [m.name for m in all_modules_ordered_by_type if m.configs], 'module_types': MODULE_TYPES, } diff --git a/scripts/settings/src/App.tsx b/scripts/settings/src/App.tsx index 126836c..1b12ba9 100644 --- a/scripts/settings/src/App.tsx +++ b/scripts/settings/src/App.tsx @@ -81,7 +81,12 @@ function FileDrop({ setYamlFile }) { function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues }: { stepType: string, setEnabledModules: any, enabledModules: any, configValues: any }) { const [showError, setShowError] = useState(false); const [activeId, setActiveId] = useState(null); - const [items, setItems] = useState(enabledModules[stepType].map(([name, enabled]: [string, boolean]) => name)); + const [items, setItems] = useState([]); + + useEffect(() => { + setItems(enabledModules[stepType].map(([name, enabled]: [string, boolean]) => name)); + } + , [enabledModules]); const toggleModule = (event: any) => { // make sure that 'feeder' and 'formatter' types only have one value @@ -100,11 +105,12 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues } else { setShowError(false); } - let newEnabledModules = { ...enabledModules }; - newEnabledModules[stepType] = enabledModules[stepType].map(([m, enabled]: [string, boolean]) => { - return (m === name) ? [m, checked] : [m, enabled]; - } - ); + let newEnabledModules = Object.fromEntries(Object.keys(enabledModules).map((type : string) => { + return [type, enabledModules[type].map(([m, enabled]: [string, boolean]) => { + return (m === name) ? [m, checked] : [m, enabled]; + })]; + } + )); setEnabledModules(newEnabledModules); } @@ -134,17 +140,16 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues return newArray.indexOf(a[0]) - newArray.indexOf(b[0]); }) setEnabledModules(newEnabledModules); - setItems(newArray); } }; return ( <> - {stepType}s + {stepType} - Select the {stepType}s you wish to enable. You can drag and drop them to reorder them. + Select the {stepType} you wish to enable. You can drag and drop them to reorder them. {showError ? Only one {stepType} can be enabled at a time. : null} @@ -186,7 +191,7 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues export default function App() { const [yamlFile, setYamlFile] = useState(new Document()); - const [enabledModules, setEnabledModules] = useState<{}>(Object.fromEntries(module_types.map(type => [type, steps[type].map((name: string) => [name, false])]))); + const [enabledModules, setEnabledModules] = useState<{}>(Object.fromEntries(Object.keys(steps).map(type => [type, steps[type].map((name: string) => [name, false])]))); const [configValues, setConfigValues] = useState<{}>( Object.keys(modules).reduce((acc, module) => { acc[module] = {}; @@ -202,7 +207,7 @@ export default function App() { // create a yaml file from const finalYaml = { - 'steps': Object.keys(stepsConfig).reduce((acc, stepType) => { + 'steps': Object.keys(steps).reduce((acc, stepType) => { acc[stepType] = stepsConfig[stepType].filter(([name, enabled]: [string, boolean]) => enabled).map(([name, enabled]: [string, boolean]) => name); return acc; }, {}) @@ -257,10 +262,27 @@ export default function App() { let settings = yamlFile.toJS(); // make a deep copy of settings - let newEnabledModules = Object.keys(settings['steps']).map((step: string) => { - return settings['steps'][step]; - }).flat(); - newEnabledModules = newEnabledModules.filter((m: string, i: number) => newEnabledModules.indexOf(m) === i); + let stepSettings = settings['steps']; + let newEnabledModules = Object.fromEntries(Object.keys(steps).map((type: string) => { + return [type, steps[type].map((name: string) => { + return [name, stepSettings[type].indexOf(name) !== -1]; + }).sort((a, b) => { + let aIndex = stepSettings[type].indexOf(a[0]); + let bIndex = stepSettings[type].indexOf(b[0]); + if (aIndex === -1 && bIndex === -1) { + return a - b; + } + if (bIndex === -1) { + return -1; + } + if (aIndex === -1) { + return 1; + } + return aIndex - bIndex; + })]; + }).sort((a, b) => { + return module_types.indexOf(a[0]) - module_types.indexOf(b[0]); + })); setEnabledModules(newEnabledModules); }, [yamlFile]); diff --git a/scripts/settings/src/StepCard.tsx b/scripts/settings/src/StepCard.tsx index d401d25..870926a 100644 --- a/scripts/settings/src/StepCard.tsx +++ b/scripts/settings/src/StepCard.tsx @@ -20,7 +20,7 @@ import { MenuItem, FormControl, FormControlLabel, - InputLabel, + Textarea, FormHelperText, TextField, Stack, @@ -141,13 +141,15 @@ function ConfigPanel({ module, open, setOpen, configValues }: { module: any, ope const value = configValues[module.name][config_value] || config_args.default; return ( - {config_display_name} + {config_display_name} - { config_args.type === 'bool' ? + { config_args.type === 'bool' ? + { setConfigValue(config_value, e.target.checked); }} + />} label={config_args.help} /> : ( @@ -167,16 +169,16 @@ function ConfigPanel({ module, open, setOpen, configValues }: { module: any, ope : ( config_args.type === 'json_loader' ? - { try { - val = JSON.parse(e.target.value); + let val = JSON.parse(e.target.value); setConfigValue(config_value, val); } catch (e) { console.log(e); } } - } type='text' /> + } /> : { @@ -185,8 +187,10 @@ function ConfigPanel({ module, open, setOpen, configValues }: { module: any, ope ) ) } - {config_args.help} - + {config_args.type !== 'bool' && ( + {config_args.help} + )} + ); })} diff --git a/scripts/settings/src/schema.json b/scripts/settings/src/schema.json index e4f470d..87ec55b 100644 --- a/scripts/settings/src/schema.json +++ b/scripts/settings/src/schema.json @@ -799,6 +799,7 @@ "configs": { "timeout": { "default": 15, + "type": "int", "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually." }, "if_not_archived_within": { @@ -826,6 +827,7 @@ "configs": { "timeout": { "default": 15, + "type": "int", "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually." }, "if_not_archived_within": { @@ -1046,18 +1048,22 @@ "configs": { "width": { "default": 1280, + "type": "int", "help": "width of the screenshots" }, "height": { "default": 720, + "type": "int", "help": "height of the screenshots" }, "timeout": { "default": 60, + "type": "int", "help": "timeout for taking the screenshot" }, "sleep_before_screenshot": { "default": 4, + "type": "int", "help": "seconds to wait for the pages to load before taking screenshot" }, "http_proxy": { @@ -1066,29 +1072,35 @@ }, "save_to_pdf": { "default": false, + "type": "bool", "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter" }, "print_options": { "default": {}, - "help": "options to pass to the pdf printer" + "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information", + "type": "json_loader" } } }, "configs": { "width": { "default": 1280, + "type": "int", "help": "width of the screenshots" }, "height": { "default": 720, + "type": "int", "help": "height of the screenshots" }, "timeout": { "default": 60, + "type": "int", "help": "timeout for taking the screenshot" }, "sleep_before_screenshot": { "default": 4, + "type": "int", "help": "seconds to wait for the pages to load before taking screenshot" }, "http_proxy": { @@ -1097,11 +1109,13 @@ }, "save_to_pdf": { "default": false, + "type": "bool", "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter" }, "print_options": { "default": {}, - "help": "options to pass to the pdf printer" + "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information", + "type": "json_loader" } } }, @@ -2007,13 +2021,13 @@ } }, "steps": { - "feeder": [ + "feeders": [ "cli_feeder", "gsheet_feeder", "atlos_feeder", "csv_feeder" ], - "extractor": [ + "extractors": [ "wayback_extractor_enricher", "wacz_extractor_enricher", "instagram_api_extractor", @@ -2025,7 +2039,7 @@ "vk_extractor", "telegram_extractor" ], - "enricher": [ + "enrichers": [ "wayback_extractor_enricher", "wacz_extractor_enricher", "metadata_enricher", @@ -2038,20 +2052,20 @@ "ssl_enricher", "hash_enricher" ], - "database": [ + "databases": [ "console_db", "atlos_db", "api_db", "csv_db", "gsheet_db" ], - "storage": [ + "storages": [ "local_storage", "gdrive_storage", "atlos_storage", "s3_storage" ], - "formatter": [ + "formatters": [ "html_formatter", "mute_formatter" ] diff --git a/src/auto_archiver/modules/screenshot_enricher/__manifest__.py b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py index 9829844..c6a196c 100644 --- a/src/auto_archiver/modules/screenshot_enricher/__manifest__.py +++ b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py @@ -6,13 +6,25 @@ "python": ["loguru", "selenium"], }, "configs": { - "width": {"default": 1280, "help": "width of the screenshots"}, - "height": {"default": 720, "help": "height of the screenshots"}, - "timeout": {"default": 60, "help": "timeout for taking the screenshot"}, - "sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"}, + "width": {"default": 1280, + "type": "int", + "help": "width of the screenshots"}, + "height": {"default": 720, + "type": "int", + "help": "height of the screenshots"}, + "timeout": {"default": 60, + "type": "int", + "help": "timeout for taking the screenshot"}, + "sleep_before_screenshot": {"default": 4, + "type": "int", + "help": "seconds to wait for the pages to load before taking screenshot"}, "http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"}, - "save_to_pdf": {"default": False, "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"}, - "print_options": {"default": {}, "help": "options to pass to the pdf printer"} + "save_to_pdf": {"default": False, + "type": "bool", + "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"}, + "print_options": {"default": {}, + "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information", + "type": "json_loader"}, }, "description": """ Captures screenshots and optionally saves web pages as PDFs using a WebDriver. diff --git a/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py b/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py index 38a5610..62a7e8a 100644 --- a/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py +++ b/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py @@ -9,6 +9,7 @@ "configs": { "timeout": { "default": 15, + "type": "int", "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually.", }, "if_not_archived_within": {