kopia lustrzana https://github.com/bellingcat/auto-archiver
Fully-working settings page editor
Patrick Robertson
rodzic
bb961b131c
commit
70d89c71ce
|
|
@ -1,7 +1,14 @@
|
|||
import os
|
||||
import json
|
||||
|
||||
from auto_archiver.core.module import ModuleFactory
|
||||
from auto_archiver.core.consts import MODULE_TYPES
|
||||
import jinja2
|
||||
|
||||
|
||||
class SchemaEncoder(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
if isinstance(obj, set):
|
||||
return list(obj)
|
||||
return json.JSONEncoder.default(self, obj)
|
||||
|
||||
# Get available modules
|
||||
module_factory = ModuleFactory()
|
||||
|
|
@ -13,73 +20,22 @@ for module in available_modules:
|
|||
for type in module.manifest.get('type', []):
|
||||
modules_by_type.setdefault(type, []).append(module)
|
||||
|
||||
|
||||
module_sections = ""
|
||||
# Add module sections
|
||||
for module_type in MODULE_TYPES:
|
||||
module_sections += f"<div class='module-section'><h3>{module_type.title()}s</h3>"
|
||||
# make this section in rows, max 8 modules per row
|
||||
for module in modules_by_type[module_type]:
|
||||
|
||||
module_name = module.name
|
||||
module_sections += f"""
|
||||
<div style="display:inline-block; width: 12.5%;">
|
||||
<input type="checkbox" id="{module.name}" name="{module.name}" onclick="toggleModuleConfig(this, '{module.name}')">
|
||||
<label for="{module.name}">{module.display_name} <a href="#{module.name}-config" id="{module.name}-config-link" style="display:none;">(configure)</a></label>
|
||||
</div>
|
||||
"""
|
||||
module_sections += "</div>"
|
||||
|
||||
# Add module configuration sections
|
||||
|
||||
all_modules_ordered_by_type = sorted(available_modules, key=lambda x: (MODULE_TYPES.index(x.type[0]), not x.requires_setup))
|
||||
|
||||
module_configs = ""
|
||||
output_schame = {
|
||||
'modules': dict((module.name,
|
||||
{
|
||||
'name': module.name,
|
||||
'display_name': module.display_name,
|
||||
'manifest': module.manifest,
|
||||
'configs': module.configs or None
|
||||
}
|
||||
) for module in all_modules_ordered_by_type),
|
||||
'steps': dict((module_type, [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES),
|
||||
'configs': [m.name for m in all_modules_ordered_by_type if m.configs],
|
||||
'module_types': MODULE_TYPES,
|
||||
}
|
||||
|
||||
for module in all_modules_ordered_by_type:
|
||||
if not module.configs:
|
||||
continue
|
||||
module_configs += f"<div id='{module.name}-config' class='module-config'><h3>{module.display_name} Configuration</h3>"
|
||||
for option, value in module.configs.items():
|
||||
# create a human readable label
|
||||
option = option.replace('_', ' ').title()
|
||||
|
||||
# type - if value has 'choices', then it's a select
|
||||
module_configs += "<div class='config-option'>"
|
||||
if 'choices' in value:
|
||||
module_configs += f"""
|
||||
<label for="{module.name}-{option}">{option}</label>
|
||||
<select id="{module.name}-{option}" name="{module.name}-{option}">
|
||||
"""
|
||||
for choice in value['choices']:
|
||||
module_configs += f"<option value='{choice}'>{choice}</option>"
|
||||
module_configs += "</select>"
|
||||
elif value.get('type') == 'bool' or isinstance(value.get('default', None), bool):
|
||||
module_configs += f"""
|
||||
<input type="checkbox" id="{module.name}-{option}" name="{module.name}-{option}">
|
||||
<label for="{module.name}-{option}">{option}</label>
|
||||
"""
|
||||
else:
|
||||
module_configs += f"""
|
||||
<label for="{module.name}-{option}">{option}</label>
|
||||
<input type="text" id="{module.name}-{option}" name="{module.name}-{option}">
|
||||
"""
|
||||
# add help text
|
||||
if 'help' in value:
|
||||
module_configs += f"<div class='help'>{value.get('help')}</div>"
|
||||
module_configs += "</div>"
|
||||
module_configs += "</div>"
|
||||
|
||||
# format the settings.html jinja page with the module sections and module configuration sections
|
||||
settings_page = "settings.html"
|
||||
template_loader = jinja2.FileSystemLoader(searchpath="./")
|
||||
template_env = jinja2.Environment(loader=template_loader)
|
||||
template = template_env.get_template(settings_page)
|
||||
html_content = template.render(module_sections=module_sections, module_configs=module_configs)
|
||||
|
||||
# Write HTML content to file
|
||||
output_file = '/Users/patrick/Developer/auto-archiver/scripts/settings_page.html'
|
||||
output_file = 'schema.json'
|
||||
with open(output_file, 'w') as file:
|
||||
file.write(html_content)
|
||||
|
||||
print(f"Settings page generated at {output_file}")
|
||||
json.dump(output_schame, file, indent=4, cls=SchemaEncoder)
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Module Settings</title>
|
||||
<style>
|
||||
.module-section { margin-bottom: 20px; }
|
||||
.module-config { display: none; }
|
||||
.help {color: #333; font-size: 0.8em; margin-left: 10px;}
|
||||
.config-option { margin-bottom: 10px; }
|
||||
</style>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/js-yaml/4.1.0/js-yaml.min.js"></script>
|
||||
<script>
|
||||
function toggleModuleConfig(elem, moduleName) {
|
||||
var configSection = document.getElementById(moduleName + '-config');
|
||||
configSection.style.display = elem.checked ? 'block' : 'none';
|
||||
}
|
||||
|
||||
function loadSettings() {
|
||||
var settingsFile = document.getElementById('settings-file').value || '/Users/patrick/Developer/auto-archiver/secrets/orchestration.yaml';
|
||||
fetch(settingsFile)
|
||||
.then(response => response.text())
|
||||
.then(yamlText => {
|
||||
var config = jsyaml.load(yamlText);
|
||||
updateSettings(config);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('Error loading settings file:', error);
|
||||
alert('Failed to load settings file.');
|
||||
});
|
||||
}
|
||||
|
||||
function updateSettings(config) {
|
||||
for (var moduleName in config) {
|
||||
var moduleCheckbox = document.getElementById(moduleName);
|
||||
if (moduleCheckbox) {
|
||||
moduleCheckbox.checked = true;
|
||||
toggleModuleConfig(moduleName);
|
||||
var moduleConfig = config[moduleName].options;
|
||||
for (var option in moduleConfig) {
|
||||
var input = document.getElementById(moduleName + '-' + option);
|
||||
if (input) {
|
||||
if (input.type === 'checkbox') {
|
||||
input.checked = moduleConfig[option];
|
||||
} else if (input.type === 'select-one') {
|
||||
input.value = moduleConfig[option];
|
||||
} else {
|
||||
input.value = moduleConfig[option];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function saveSettings() {
|
||||
var formData = new FormData(document.querySelector('form'));
|
||||
var config = {};
|
||||
|
||||
formData.forEach((value, key) => {
|
||||
var [moduleName, option] = key.split('-');
|
||||
if (!config[moduleName]) {
|
||||
config[moduleName] = { enabled: false, options: {} };
|
||||
}
|
||||
if (option === undefined) {
|
||||
config[moduleName].enabled = value === 'on';
|
||||
} else {
|
||||
config[moduleName].options[option] = value;
|
||||
}
|
||||
});
|
||||
|
||||
fetch('/save_settings', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(config)
|
||||
}).then(response => {
|
||||
if (response.ok) {
|
||||
alert('Settings saved successfully!');
|
||||
} else {
|
||||
alert('Failed to save settings.');
|
||||
}
|
||||
});
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Module Settings</h1>
|
||||
<form onsubmit="event.preventDefault(); saveSettings();">
|
||||
<label for="settings-file">Settings File:</label>
|
||||
<input type="text" id="settings-file" name="settings-file" value="/Users/patrick/Developer/auto-archiver/secrets/orchestration.yaml">
|
||||
<button type="button" onclick="loadSettings()">Load Settings</button>
|
||||
<h2>Steps</h2>
|
||||
|
||||
{{module_sections}}
|
||||
|
||||
<h2>Module Configuration Section</h2>
|
||||
{{module_configs}}
|
||||
|
||||
<button type="submit">Save Settings</button>
|
||||
</form>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
lerna-debug.log*
|
||||
|
||||
node_modules
|
||||
dist
|
||||
dist-ssr
|
||||
*.local
|
||||
|
||||
# Editor directories and files
|
||||
.vscode/*
|
||||
!.vscode/extensions.json
|
||||
.idea
|
||||
.DS_Store
|
||||
*.suo
|
||||
*.ntvs*
|
||||
*.njsproj
|
||||
*.sln
|
||||
*.sw?
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
|
||||
<meta name="viewport" content="initial-scale=1, width=device-width" />
|
||||
<!-- Fonts to support Material Design -->
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link
|
||||
rel="stylesheet"
|
||||
href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap"
|
||||
/>
|
||||
<title>Auto Archiver Settings</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
Plik diff jest za duży
Load Diff
|
|
@ -0,0 +1,29 @@
|
|||
{
|
||||
"name": "material-ui-vite-ts",
|
||||
"private": true,
|
||||
"version": "5.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc && vite build",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"@emotion/react": "latest",
|
||||
"@emotion/styled": "latest",
|
||||
"@mui/icons-material": "latest",
|
||||
"@mui/material": "latest",
|
||||
"react": "latest",
|
||||
"react-dom": "latest",
|
||||
"react-markdown": "^10.0.0",
|
||||
"yaml": "^2.7.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "latest",
|
||||
"@types/react-dom": "latest",
|
||||
"@vitejs/plugin-react": "latest",
|
||||
"typescript": "latest",
|
||||
"vite": "latest",
|
||||
"vite-plugin-singlefile": "^2.1.0"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
|
||||
|
Po Szerokość: | Wysokość: | Rozmiar: 1.5 KiB |
|
|
@ -0,0 +1,364 @@
|
|||
import * as React from 'react';
|
||||
import { useEffect, useState } from 'react';
|
||||
import Container from '@mui/material/Container';
|
||||
import Typography from '@mui/material/Typography';
|
||||
import Box from '@mui/material/Box';
|
||||
import Link from '@mui/material/Link';
|
||||
import { modules, steps, configs, module_types } from './schema.json';
|
||||
import {
|
||||
Checkbox,
|
||||
Select,
|
||||
MenuItem,
|
||||
FormControl,
|
||||
FormControlLabel,
|
||||
InputLabel,
|
||||
FormHelperText,
|
||||
Stack,
|
||||
TextField,
|
||||
Card,
|
||||
CardContent,
|
||||
CardActions,
|
||||
Button,
|
||||
Dialog,
|
||||
DialogTitle,
|
||||
DialogContent,
|
||||
} from '@mui/material';
|
||||
import Grid from '@mui/material/Grid2';
|
||||
|
||||
import Accordion from '@mui/material/Accordion';
|
||||
import AccordionDetails from '@mui/material/AccordionDetails';
|
||||
import AccordionSummary from '@mui/material/AccordionSummary';
|
||||
import ArrowDropDownIcon from '@mui/icons-material/ArrowDropDown';
|
||||
import ReactMarkdown from 'react-markdown';
|
||||
import { parseDocument, ParsedNode, Document } from 'yaml'
|
||||
import { set } from 'yaml/dist/schema/yaml-1.1/set';
|
||||
|
||||
Object.defineProperty(String.prototype, 'capitalize', {
|
||||
value: function() {
|
||||
return this.charAt(0).toUpperCase() + this.slice(1);
|
||||
},
|
||||
enumerable: false
|
||||
});
|
||||
|
||||
function FileDrop({ setYamlFile }) {
|
||||
|
||||
const [showError, setShowError] = useState(false);
|
||||
const [label, setLabel] = useState("Drag and drop your orchestration.yaml file here, or click to select a file.");
|
||||
|
||||
function openYAMLFile(event: any) {
|
||||
let file = event.target.files[0];
|
||||
if (file.type !== 'application/x-yaml') {
|
||||
setShowError(true);
|
||||
setLabel("Invalid type, only YAML files are accepted.")
|
||||
return;
|
||||
}
|
||||
let reader = new FileReader();
|
||||
reader.onload = function(e) {
|
||||
let contents = e.target.result;
|
||||
try {
|
||||
let document = parseDocument(contents);
|
||||
if (document.errors.length > 0) {
|
||||
// not a valid yaml file
|
||||
setShowError(true);
|
||||
setLabel("Invalid file. Make sure your Orchestration is a valid YAML file with a 'steps' section in it.")
|
||||
return;
|
||||
} else {
|
||||
setShowError(false);
|
||||
setLabel("File loaded successfully.")
|
||||
}
|
||||
setYamlFile(document);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
reader.readAsText(file);
|
||||
}
|
||||
return (
|
||||
<>
|
||||
<div style={{width:'100%', border:'dashed', textAlign:'center', borderWidth:'1px', padding:'20px'}}>
|
||||
|
||||
<input name="file" type="file" accept=".yaml" onChange={openYAMLFile} />
|
||||
<Typography style={{marginTop:'20px' }} variant="body1" color={showError ? 'error' : ''} >
|
||||
{label}
|
||||
</Typography>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
function ModuleCheckbox({ module, toggleModule, enabledModules, configValues }: { module: object, toggleModule: any, enabledModules: any, configValues: any }) {
|
||||
let name = module.name;
|
||||
const [helpOpen, setHelpOpen] = useState(false);
|
||||
const [configOpen, setConfigOpen] = useState(false);
|
||||
if (name == 'metadata_enricher') {
|
||||
console.log("hi");
|
||||
}
|
||||
return (
|
||||
<>
|
||||
<Card>
|
||||
<CardContent>
|
||||
<FormControlLabel
|
||||
control={<Checkbox id={name} onClick={toggleModule} checked={enabledModules.includes(name)} />}
|
||||
label={module.display_name} />
|
||||
</CardContent>
|
||||
<CardActions>
|
||||
<Button size="small" onClick={() => setHelpOpen(true)}>Help</Button>
|
||||
{enabledModules.includes(name) && module.configs && name != 'cli_feeder' ? (
|
||||
<Button size="small" onClick={() => setConfigOpen(true)}>Configure</Button>
|
||||
) : null}
|
||||
</CardActions>
|
||||
</Card>
|
||||
<Dialog
|
||||
open={helpOpen}
|
||||
onClose={() => setHelpOpen(false)}
|
||||
maxWidth="lg"
|
||||
>
|
||||
<DialogTitle>
|
||||
{module.display_name}
|
||||
</DialogTitle>
|
||||
<DialogContent>
|
||||
<ReactMarkdown>
|
||||
{module.manifest.description.split("\n").map((line: string) => line.trim()).join("\n")}
|
||||
</ReactMarkdown>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
{module.configs && name != 'cli_feeder' && <ConfigPanel module={module} open={configOpen} setOpen={setConfigOpen} configValues={configValues[module.name]} />}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
function ConfigPanel({ module, open, setOpen, configValues }: { module: any, open: boolean, setOpen: any, configValues: any }) {
|
||||
return (
|
||||
<>
|
||||
<Dialog
|
||||
key={module}
|
||||
open={open}
|
||||
onClose={() => setOpen(false)}
|
||||
maxWidth="lg"
|
||||
>
|
||||
<DialogTitle>
|
||||
{module.display_name}
|
||||
</DialogTitle>
|
||||
<DialogContent>
|
||||
<Stack key={module} direction="column" spacing={1}>
|
||||
{Object.keys(module.configs).map((config_value: any) => {
|
||||
let config_args = module.configs[config_value];
|
||||
let config_name = config_value.replace(/_/g," ");
|
||||
return (
|
||||
<Box key={config_value}>
|
||||
<FormControl size="small">
|
||||
{ config_args.type === 'bool' ?
|
||||
<FormControlLabel style={{ textTransform: 'capitalize'}} control={<Checkbox checked={configValues[config_value]} size="small" id={`${module}.${config_value}`} />} label={config_name} />
|
||||
:
|
||||
( config_args.type === 'int' ?
|
||||
<TextField size="small" id={`${module}.${config_value}`} label={config_name.capitalize()} value={configValues[config_value]} type="number" />
|
||||
:
|
||||
(
|
||||
config_args.choices !== undefined ?
|
||||
<>
|
||||
<InputLabel>{config_name}</InputLabel>
|
||||
<Select size="small" id={`${module}.${config_value}`}
|
||||
defaultValue={config_args.default} value={configValues[config_value] || ''}>
|
||||
{config_args.choices.map((choice: any) => {
|
||||
return (
|
||||
<MenuItem key={`${module}.${config_value}.${choice}`}
|
||||
value={choice} selected={config_args.default === choice}>{choice}</MenuItem>
|
||||
);
|
||||
})}
|
||||
</Select>
|
||||
</>
|
||||
:
|
||||
<TextField size="small" id={`${module}.${config_value}`} value={configValues[config_value] || ''} label={config_name.capitalize()} />
|
||||
)
|
||||
)
|
||||
}
|
||||
<FormHelperText style={{ textTransform: 'capitalize'}}>{config_args.help}</FormHelperText>
|
||||
</FormControl>
|
||||
</Box>
|
||||
);
|
||||
})}
|
||||
</Stack>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function ModuleTypes({ stepType, toggleModule, enabledModules, configValues }: { stepType: string, toggleModule: any, enabledModules: any, configValues: any }) {
|
||||
const [showError, setShowError] = useState(false);
|
||||
|
||||
const _toggleModule = (event: any) => {
|
||||
// make sure that 'feeder' and 'formatter' types only have one value
|
||||
let name = event.target.id;
|
||||
if (stepType === 'feeder' || stepType === 'formatter') {
|
||||
let checked = event.target.checked;
|
||||
// check how many modules of this type are enabled
|
||||
let modules = steps[stepType].filter((m: string) => (m !== name && enabledModules.includes(m)) || (checked && m === name));
|
||||
if (modules.length > 1) {
|
||||
setShowError(true);
|
||||
} else {
|
||||
setShowError(false);
|
||||
}
|
||||
} else {
|
||||
setShowError(false);
|
||||
}
|
||||
toggleModule(event);
|
||||
}
|
||||
return (
|
||||
<>
|
||||
<Typography id={stepType} variant="h6" style={{ textTransform: 'capitalize' }} >
|
||||
{stepType}s
|
||||
</Typography>
|
||||
{showError ? <Typography variant="body1" color="error" >Only one {stepType} can be enabled at a time.</Typography> : null}
|
||||
<Grid container spacing={1} key={stepType}>
|
||||
{steps[stepType].map((name: string) => {
|
||||
let m = modules[name];
|
||||
return (
|
||||
<Grid key={name} size={{ xs: 6, sm: 4, md: 3 }}>
|
||||
<ModuleCheckbox key={name} module={m} toggleModule={_toggleModule} enabledModules={enabledModules} configValues={configValues} />
|
||||
</Grid>
|
||||
);
|
||||
})}
|
||||
</Grid>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
export default function App() {
|
||||
const [yamlFile, setYamlFile] = useState<Document>(new Document());
|
||||
const [enabledModules, setEnabledModules] = useState<[]>([]);
|
||||
const [configValues, setConfigValues] = useState<{}>(
|
||||
Object.keys(modules).reduce((acc, module) => {
|
||||
acc[module] = {};
|
||||
return acc;
|
||||
}, {})
|
||||
);
|
||||
|
||||
const saveSettings = function(copy: boolean = false) {
|
||||
// edit the yamlFile
|
||||
|
||||
// generate the steps config
|
||||
let stepsConfig = {}
|
||||
module_types.forEach((stepType: string) => {
|
||||
stepsConfig[stepType] = enabledModules.filter((m: string) => steps[stepType].includes(m));
|
||||
}
|
||||
);
|
||||
|
||||
// create a yaml file from
|
||||
const finalYaml = {
|
||||
'steps': stepsConfig
|
||||
};
|
||||
|
||||
Object.keys(configValues).map((module: string) => {
|
||||
let module_values = configValues[module];
|
||||
if (module_values) {
|
||||
finalYaml[module] = module_values;
|
||||
}
|
||||
});
|
||||
let newFile = new Document(finalYaml);
|
||||
if (copy) {
|
||||
navigator.clipboard.writeText(String(newFile)).then(() => {
|
||||
alert("Settings copied to clipboard.");
|
||||
});
|
||||
} else {
|
||||
// offer the file for download
|
||||
const blob = new Blob([String(newFile)], { type: 'application/x-yaml' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = 'orchestration.yaml';
|
||||
a.click();
|
||||
}
|
||||
}
|
||||
|
||||
const toggleModule = function (event: any) {
|
||||
let module = event.target.id;
|
||||
let checked = event.target.checked
|
||||
|
||||
if (checked) {
|
||||
setEnabledModules([...enabledModules, module]);
|
||||
} else {
|
||||
setEnabledModules(enabledModules.filter((m: string) => m !== module));
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
// load the configs, and set the default values if they exist
|
||||
let newConfigValues = {};
|
||||
Object.keys(modules).map((module: string) => {
|
||||
let m = modules[module];
|
||||
let configs = m.configs;
|
||||
if (!configs) {
|
||||
return;
|
||||
}
|
||||
newConfigValues[module] = {};
|
||||
Object.keys(configs).map((config: string) => {
|
||||
let config_args = configs[config];
|
||||
if (config_args.default !== undefined) {
|
||||
newConfigValues[module][config] = config_args.default;
|
||||
}
|
||||
});
|
||||
})
|
||||
setConfigValues(newConfigValues);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!yamlFile || yamlFile.contents == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
let settings = yamlFile.toJS();
|
||||
// make a deep copy of settings
|
||||
let newEnabledModules = Object.keys(settings['steps']).map((step: string) => {
|
||||
return settings['steps'][step];
|
||||
}).flat();
|
||||
newEnabledModules = newEnabledModules.filter((m: string, i: number) => newEnabledModules.indexOf(m) === i);
|
||||
setEnabledModules(newEnabledModules);
|
||||
}, [yamlFile]);
|
||||
|
||||
|
||||
|
||||
return (
|
||||
<Container maxWidth="lg">
|
||||
<Box sx={{ my: 4 }}>
|
||||
<Typography variant="h2" >
|
||||
Auto Archiver Settings
|
||||
</Typography>
|
||||
<Box sx={{ my: 4 }}>
|
||||
<Typography variant="h5" >
|
||||
1. Select your <pre style={{display:'inline'}}>orchestration.yaml</pre> settings file.
|
||||
</Typography>
|
||||
<FileDrop setYamlFile={setYamlFile}/>
|
||||
</Box>
|
||||
<Box sx={{ my: 4 }}>
|
||||
<Typography variant="h5" >
|
||||
2. Choose the Modules you wish to enable/disable
|
||||
</Typography>
|
||||
{Object.keys(steps).map((stepType: string) => {
|
||||
return (
|
||||
<ModuleTypes key={stepType} stepType={stepType} toggleModule={toggleModule} enabledModules={enabledModules} configValues={configValues} />
|
||||
);
|
||||
})}
|
||||
</Box>
|
||||
<Box sx={{ my: 4 }}>
|
||||
<Typography variant="h5" >
|
||||
3. Configure your Enabled Modules
|
||||
</Typography>
|
||||
<Typography variant="body1" >
|
||||
Next to each module you've enabled, you can click 'Configure' to set the module's settings.
|
||||
</Typography>
|
||||
</Box>
|
||||
<Box sx={{ my: 4 }}>
|
||||
<Typography variant="h5" >
|
||||
4. Save your settings
|
||||
</Typography>
|
||||
<Button variant="contained" color="primary" onClick={() => saveSettings(true)}>Copy Settings to Clipboard</Button>
|
||||
<Button variant="contained" color="primary" onClick={() => saveSettings()}>Save Settings to File</Button>
|
||||
</Box>
|
||||
</Box>
|
||||
</Container>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
import * as React from 'react';
|
||||
import Link from '@mui/material/Link';
|
||||
import SvgIcon, { SvgIconProps } from '@mui/material/SvgIcon';
|
||||
import Typography from '@mui/material/Typography';
|
||||
|
||||
function LightBulbIcon(props: SvgIconProps) {
|
||||
return (
|
||||
<SvgIcon {...props}>
|
||||
<path d="M9 21c0 .55.45 1 1 1h4c.55 0 1-.45 1-1v-1H9v1zm3-19C8.14 2 5 5.14 5 9c0 2.38 1.19 4.47 3 5.74V17c0 .55.45 1 1 1h6c.55 0 1-.45 1-1v-2.26c1.81-1.27 3-3.36 3-5.74 0-3.86-3.14-7-7-7zm2.85 11.1l-.85.6V16h-4v-2.3l-.85-.6C7.8 12.16 7 10.63 7 9c0-2.76 2.24-5 5-5s5 2.24 5 5c0 1.63-.8 3.16-2.15 4.1z" />
|
||||
</SvgIcon>
|
||||
);
|
||||
}
|
||||
|
||||
export default function ProTip() {
|
||||
return (
|
||||
<Typography sx={{ mt: 6, mb: 3, color: 'text.secondary' }}>
|
||||
<LightBulbIcon sx={{ mr: 1, verticalAlign: 'middle' }} />
|
||||
{'Pro tip: See more '}
|
||||
<Link href="https://mui.com/material-ui/getting-started/templates/">templates</Link>
|
||||
{' in the Material UI documentation.'}
|
||||
</Typography>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
import * as React from 'react';
|
||||
import * as ReactDOM from 'react-dom/client';
|
||||
import { ThemeProvider } from '@mui/material/styles';
|
||||
import { CssBaseline } from '@mui/material';
|
||||
import theme from './theme';
|
||||
import App from './App';
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||
<React.StrictMode>
|
||||
<ThemeProvider theme={theme}>
|
||||
<CssBaseline />
|
||||
<App />
|
||||
</ThemeProvider>
|
||||
</React.StrictMode>,
|
||||
);
|
||||
Plik diff jest za duży
Load Diff
|
|
@ -0,0 +1,20 @@
|
|||
import { createTheme } from '@mui/material/styles';
|
||||
import { red } from '@mui/material/colors';
|
||||
|
||||
// A custom theme for this app
|
||||
const theme = createTheme({
|
||||
cssVariables: true,
|
||||
palette: {
|
||||
primary: {
|
||||
main: '#556cd6',
|
||||
},
|
||||
secondary: {
|
||||
main: '#19857b',
|
||||
},
|
||||
error: {
|
||||
main: red.A400,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
export default theme;
|
||||
|
|
@ -0,0 +1 @@
|
|||
/// <reference types="vite/client" />
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"useDefineForClassFields": true,
|
||||
"lib": ["DOM", "DOM.Iterable", "ESNext"],
|
||||
"allowJs": false,
|
||||
"skipLibCheck": true,
|
||||
"esModuleInterop": false,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"strict": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Node",
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"noEmit": true,
|
||||
"jsx": "react-jsx"
|
||||
},
|
||||
"include": ["src"],
|
||||
"references": [{ "path": "./tsconfig.node.json" }]
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Node",
|
||||
"allowSyntheticDefaultImports": true
|
||||
},
|
||||
"include": ["vite.config.ts"]
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
import { defineConfig } from 'vite';
|
||||
import react from '@vitejs/plugin-react';
|
||||
import { viteSingleFile } from "vite-plugin-singlefile"
|
||||
|
||||
// https://vite.dev/config/
|
||||
export default defineConfig({
|
||||
plugins: [react(), viteSingleFile()],
|
||||
});
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
},
|
||||
'entry_point': 'csv_db::CSVDb',
|
||||
"configs": {
|
||||
"csv_file": {"default": "db.csv", "help": "CSV file name"}
|
||||
"csv_file": {"default": "db.csv", "help": "CSV file name to save metadata to"},
|
||||
},
|
||||
"description": """
|
||||
Handles exporting archival results to a CSV file.
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@
|
|||
"bin": [""]
|
||||
},
|
||||
"configs": {
|
||||
"detect_thumbnails": {"default": True, "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'"}
|
||||
"detect_thumbnails": {"default": True,
|
||||
"help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'",
|
||||
"type": "bool"},
|
||||
},
|
||||
"description": """ """,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@
|
|||
},
|
||||
'entry_point': 'ssl_enricher::SSLEnricher',
|
||||
"configs": {
|
||||
"skip_when_nothing_archived": {"default": True, "help": "if true, will skip enriching when no media is archived"},
|
||||
"skip_when_nothing_archived": {"default": True,
|
||||
"type": 'bool',
|
||||
"help": "if true, will skip enriching when no media is archived"},
|
||||
},
|
||||
"description": """
|
||||
Retrieves SSL certificate information for a domain and stores it as a file.
|
||||
|
|
|
|||
|
|
@ -17,11 +17,17 @@
|
|||
"configs": {
|
||||
"profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."},
|
||||
"docker_commands": {"default": None, "help":"if a custom docker invocation is needed"},
|
||||
"timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"},
|
||||
"extract_media": {"default": False, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."},
|
||||
"extract_screenshot": {"default": True, "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."},
|
||||
"timeout": {"default": 120, "help": "timeout for WACZ generation in seconds", "type": "int"},
|
||||
"extract_media": {"default": False,
|
||||
"type": 'bool',
|
||||
"help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."
|
||||
},
|
||||
"extract_screenshot": {"default": True,
|
||||
"type": 'bool',
|
||||
"help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."
|
||||
},
|
||||
"socks_proxy_host": {"default": None, "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host"},
|
||||
"socks_proxy_port": {"default": None, "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"},
|
||||
"socks_proxy_port": {"default": None, "type":"int", "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"},
|
||||
"proxy_server": {"default": None, "help": "SOCKS server proxy URL, in development"},
|
||||
},
|
||||
"description": """
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue