kopia lustrzana https://github.com/bellingcat/auto-archiver
Fully-working settings page editor
Patrick Robertson
rodzic
bb961b131c
commit
70d89c71ce
|
|
@ -1,7 +1,14 @@
|
||||||
import os
|
import json
|
||||||
|
|
||||||
from auto_archiver.core.module import ModuleFactory
|
from auto_archiver.core.module import ModuleFactory
|
||||||
from auto_archiver.core.consts import MODULE_TYPES
|
from auto_archiver.core.consts import MODULE_TYPES
|
||||||
import jinja2
|
|
||||||
|
|
||||||
|
class SchemaEncoder(json.JSONEncoder):
|
||||||
|
def default(self, obj):
|
||||||
|
if isinstance(obj, set):
|
||||||
|
return list(obj)
|
||||||
|
return json.JSONEncoder.default(self, obj)
|
||||||
|
|
||||||
# Get available modules
|
# Get available modules
|
||||||
module_factory = ModuleFactory()
|
module_factory = ModuleFactory()
|
||||||
|
|
@ -13,73 +20,22 @@ for module in available_modules:
|
||||||
for type in module.manifest.get('type', []):
|
for type in module.manifest.get('type', []):
|
||||||
modules_by_type.setdefault(type, []).append(module)
|
modules_by_type.setdefault(type, []).append(module)
|
||||||
|
|
||||||
|
|
||||||
module_sections = ""
|
|
||||||
# Add module sections
|
|
||||||
for module_type in MODULE_TYPES:
|
|
||||||
module_sections += f"<div class='module-section'><h3>{module_type.title()}s</h3>"
|
|
||||||
# make this section in rows, max 8 modules per row
|
|
||||||
for module in modules_by_type[module_type]:
|
|
||||||
|
|
||||||
module_name = module.name
|
|
||||||
module_sections += f"""
|
|
||||||
<div style="display:inline-block; width: 12.5%;">
|
|
||||||
<input type="checkbox" id="{module.name}" name="{module.name}" onclick="toggleModuleConfig(this, '{module.name}')">
|
|
||||||
<label for="{module.name}">{module.display_name} <a href="#{module.name}-config" id="{module.name}-config-link" style="display:none;">(configure)</a></label>
|
|
||||||
</div>
|
|
||||||
"""
|
|
||||||
module_sections += "</div>"
|
|
||||||
|
|
||||||
# Add module configuration sections
|
|
||||||
|
|
||||||
all_modules_ordered_by_type = sorted(available_modules, key=lambda x: (MODULE_TYPES.index(x.type[0]), not x.requires_setup))
|
all_modules_ordered_by_type = sorted(available_modules, key=lambda x: (MODULE_TYPES.index(x.type[0]), not x.requires_setup))
|
||||||
|
|
||||||
module_configs = ""
|
output_schame = {
|
||||||
|
'modules': dict((module.name,
|
||||||
|
{
|
||||||
|
'name': module.name,
|
||||||
|
'display_name': module.display_name,
|
||||||
|
'manifest': module.manifest,
|
||||||
|
'configs': module.configs or None
|
||||||
|
}
|
||||||
|
) for module in all_modules_ordered_by_type),
|
||||||
|
'steps': dict((module_type, [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES),
|
||||||
|
'configs': [m.name for m in all_modules_ordered_by_type if m.configs],
|
||||||
|
'module_types': MODULE_TYPES,
|
||||||
|
}
|
||||||
|
|
||||||
for module in all_modules_ordered_by_type:
|
output_file = 'schema.json'
|
||||||
if not module.configs:
|
|
||||||
continue
|
|
||||||
module_configs += f"<div id='{module.name}-config' class='module-config'><h3>{module.display_name} Configuration</h3>"
|
|
||||||
for option, value in module.configs.items():
|
|
||||||
# create a human readable label
|
|
||||||
option = option.replace('_', ' ').title()
|
|
||||||
|
|
||||||
# type - if value has 'choices', then it's a select
|
|
||||||
module_configs += "<div class='config-option'>"
|
|
||||||
if 'choices' in value:
|
|
||||||
module_configs += f"""
|
|
||||||
<label for="{module.name}-{option}">{option}</label>
|
|
||||||
<select id="{module.name}-{option}" name="{module.name}-{option}">
|
|
||||||
"""
|
|
||||||
for choice in value['choices']:
|
|
||||||
module_configs += f"<option value='{choice}'>{choice}</option>"
|
|
||||||
module_configs += "</select>"
|
|
||||||
elif value.get('type') == 'bool' or isinstance(value.get('default', None), bool):
|
|
||||||
module_configs += f"""
|
|
||||||
<input type="checkbox" id="{module.name}-{option}" name="{module.name}-{option}">
|
|
||||||
<label for="{module.name}-{option}">{option}</label>
|
|
||||||
"""
|
|
||||||
else:
|
|
||||||
module_configs += f"""
|
|
||||||
<label for="{module.name}-{option}">{option}</label>
|
|
||||||
<input type="text" id="{module.name}-{option}" name="{module.name}-{option}">
|
|
||||||
"""
|
|
||||||
# add help text
|
|
||||||
if 'help' in value:
|
|
||||||
module_configs += f"<div class='help'>{value.get('help')}</div>"
|
|
||||||
module_configs += "</div>"
|
|
||||||
module_configs += "</div>"
|
|
||||||
|
|
||||||
# format the settings.html jinja page with the module sections and module configuration sections
|
|
||||||
settings_page = "settings.html"
|
|
||||||
template_loader = jinja2.FileSystemLoader(searchpath="./")
|
|
||||||
template_env = jinja2.Environment(loader=template_loader)
|
|
||||||
template = template_env.get_template(settings_page)
|
|
||||||
html_content = template.render(module_sections=module_sections, module_configs=module_configs)
|
|
||||||
|
|
||||||
# Write HTML content to file
|
|
||||||
output_file = '/Users/patrick/Developer/auto-archiver/scripts/settings_page.html'
|
|
||||||
with open(output_file, 'w') as file:
|
with open(output_file, 'w') as file:
|
||||||
file.write(html_content)
|
json.dump(output_schame, file, indent=4, cls=SchemaEncoder)
|
||||||
|
|
||||||
print(f"Settings page generated at {output_file}")
|
|
||||||
|
|
@ -1,105 +0,0 @@
|
||||||
<!DOCTYPE html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
||||||
<title>Module Settings</title>
|
|
||||||
<style>
|
|
||||||
.module-section { margin-bottom: 20px; }
|
|
||||||
.module-config { display: none; }
|
|
||||||
.help {color: #333; font-size: 0.8em; margin-left: 10px;}
|
|
||||||
.config-option { margin-bottom: 10px; }
|
|
||||||
</style>
|
|
||||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/js-yaml/4.1.0/js-yaml.min.js"></script>
|
|
||||||
<script>
|
|
||||||
function toggleModuleConfig(elem, moduleName) {
|
|
||||||
var configSection = document.getElementById(moduleName + '-config');
|
|
||||||
configSection.style.display = elem.checked ? 'block' : 'none';
|
|
||||||
}
|
|
||||||
|
|
||||||
function loadSettings() {
|
|
||||||
var settingsFile = document.getElementById('settings-file').value || '/Users/patrick/Developer/auto-archiver/secrets/orchestration.yaml';
|
|
||||||
fetch(settingsFile)
|
|
||||||
.then(response => response.text())
|
|
||||||
.then(yamlText => {
|
|
||||||
var config = jsyaml.load(yamlText);
|
|
||||||
updateSettings(config);
|
|
||||||
})
|
|
||||||
.catch(error => {
|
|
||||||
console.error('Error loading settings file:', error);
|
|
||||||
alert('Failed to load settings file.');
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function updateSettings(config) {
|
|
||||||
for (var moduleName in config) {
|
|
||||||
var moduleCheckbox = document.getElementById(moduleName);
|
|
||||||
if (moduleCheckbox) {
|
|
||||||
moduleCheckbox.checked = true;
|
|
||||||
toggleModuleConfig(moduleName);
|
|
||||||
var moduleConfig = config[moduleName].options;
|
|
||||||
for (var option in moduleConfig) {
|
|
||||||
var input = document.getElementById(moduleName + '-' + option);
|
|
||||||
if (input) {
|
|
||||||
if (input.type === 'checkbox') {
|
|
||||||
input.checked = moduleConfig[option];
|
|
||||||
} else if (input.type === 'select-one') {
|
|
||||||
input.value = moduleConfig[option];
|
|
||||||
} else {
|
|
||||||
input.value = moduleConfig[option];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function saveSettings() {
|
|
||||||
var formData = new FormData(document.querySelector('form'));
|
|
||||||
var config = {};
|
|
||||||
|
|
||||||
formData.forEach((value, key) => {
|
|
||||||
var [moduleName, option] = key.split('-');
|
|
||||||
if (!config[moduleName]) {
|
|
||||||
config[moduleName] = { enabled: false, options: {} };
|
|
||||||
}
|
|
||||||
if (option === undefined) {
|
|
||||||
config[moduleName].enabled = value === 'on';
|
|
||||||
} else {
|
|
||||||
config[moduleName].options[option] = value;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
fetch('/save_settings', {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json'
|
|
||||||
},
|
|
||||||
body: JSON.stringify(config)
|
|
||||||
}).then(response => {
|
|
||||||
if (response.ok) {
|
|
||||||
alert('Settings saved successfully!');
|
|
||||||
} else {
|
|
||||||
alert('Failed to save settings.');
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Module Settings</h1>
|
|
||||||
<form onsubmit="event.preventDefault(); saveSettings();">
|
|
||||||
<label for="settings-file">Settings File:</label>
|
|
||||||
<input type="text" id="settings-file" name="settings-file" value="/Users/patrick/Developer/auto-archiver/secrets/orchestration.yaml">
|
|
||||||
<button type="button" onclick="loadSettings()">Load Settings</button>
|
|
||||||
<h2>Steps</h2>
|
|
||||||
|
|
||||||
{{module_sections}}
|
|
||||||
|
|
||||||
<h2>Module Configuration Section</h2>
|
|
||||||
{{module_configs}}
|
|
||||||
|
|
||||||
<button type="submit">Save Settings</button>
|
|
||||||
</form>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
# Logs
|
||||||
|
logs
|
||||||
|
*.log
|
||||||
|
npm-debug.log*
|
||||||
|
yarn-debug.log*
|
||||||
|
yarn-error.log*
|
||||||
|
pnpm-debug.log*
|
||||||
|
lerna-debug.log*
|
||||||
|
|
||||||
|
node_modules
|
||||||
|
dist
|
||||||
|
dist-ssr
|
||||||
|
*.local
|
||||||
|
|
||||||
|
# Editor directories and files
|
||||||
|
.vscode/*
|
||||||
|
!.vscode/extensions.json
|
||||||
|
.idea
|
||||||
|
.DS_Store
|
||||||
|
*.suo
|
||||||
|
*.ntvs*
|
||||||
|
*.njsproj
|
||||||
|
*.sln
|
||||||
|
*.sw?
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
|
||||||
|
<meta name="viewport" content="initial-scale=1, width=device-width" />
|
||||||
|
<!-- Fonts to support Material Design -->
|
||||||
|
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||||
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||||
|
<link
|
||||||
|
rel="stylesheet"
|
||||||
|
href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap"
|
||||||
|
/>
|
||||||
|
<title>Auto Archiver Settings</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div id="root"></div>
|
||||||
|
<script type="module" src="/src/main.tsx"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
Plik diff jest za duży
Load Diff
|
|
@ -0,0 +1,29 @@
|
||||||
|
{
|
||||||
|
"name": "material-ui-vite-ts",
|
||||||
|
"private": true,
|
||||||
|
"version": "5.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "vite",
|
||||||
|
"build": "tsc && vite build",
|
||||||
|
"preview": "vite preview"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@emotion/react": "latest",
|
||||||
|
"@emotion/styled": "latest",
|
||||||
|
"@mui/icons-material": "latest",
|
||||||
|
"@mui/material": "latest",
|
||||||
|
"react": "latest",
|
||||||
|
"react-dom": "latest",
|
||||||
|
"react-markdown": "^10.0.0",
|
||||||
|
"yaml": "^2.7.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/react": "latest",
|
||||||
|
"@types/react-dom": "latest",
|
||||||
|
"@vitejs/plugin-react": "latest",
|
||||||
|
"typescript": "latest",
|
||||||
|
"vite": "latest",
|
||||||
|
"vite-plugin-singlefile": "^2.1.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
|
||||||
|
Po Szerokość: | Wysokość: | Rozmiar: 1.5 KiB |
|
|
@ -0,0 +1,364 @@
|
||||||
|
import * as React from 'react';
|
||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import Container from '@mui/material/Container';
|
||||||
|
import Typography from '@mui/material/Typography';
|
||||||
|
import Box from '@mui/material/Box';
|
||||||
|
import Link from '@mui/material/Link';
|
||||||
|
import { modules, steps, configs, module_types } from './schema.json';
|
||||||
|
import {
|
||||||
|
Checkbox,
|
||||||
|
Select,
|
||||||
|
MenuItem,
|
||||||
|
FormControl,
|
||||||
|
FormControlLabel,
|
||||||
|
InputLabel,
|
||||||
|
FormHelperText,
|
||||||
|
Stack,
|
||||||
|
TextField,
|
||||||
|
Card,
|
||||||
|
CardContent,
|
||||||
|
CardActions,
|
||||||
|
Button,
|
||||||
|
Dialog,
|
||||||
|
DialogTitle,
|
||||||
|
DialogContent,
|
||||||
|
} from '@mui/material';
|
||||||
|
import Grid from '@mui/material/Grid2';
|
||||||
|
|
||||||
|
import Accordion from '@mui/material/Accordion';
|
||||||
|
import AccordionDetails from '@mui/material/AccordionDetails';
|
||||||
|
import AccordionSummary from '@mui/material/AccordionSummary';
|
||||||
|
import ArrowDropDownIcon from '@mui/icons-material/ArrowDropDown';
|
||||||
|
import ReactMarkdown from 'react-markdown';
|
||||||
|
import { parseDocument, ParsedNode, Document } from 'yaml'
|
||||||
|
import { set } from 'yaml/dist/schema/yaml-1.1/set';
|
||||||
|
|
||||||
|
Object.defineProperty(String.prototype, 'capitalize', {
|
||||||
|
value: function() {
|
||||||
|
return this.charAt(0).toUpperCase() + this.slice(1);
|
||||||
|
},
|
||||||
|
enumerable: false
|
||||||
|
});
|
||||||
|
|
||||||
|
function FileDrop({ setYamlFile }) {
|
||||||
|
|
||||||
|
const [showError, setShowError] = useState(false);
|
||||||
|
const [label, setLabel] = useState("Drag and drop your orchestration.yaml file here, or click to select a file.");
|
||||||
|
|
||||||
|
function openYAMLFile(event: any) {
|
||||||
|
let file = event.target.files[0];
|
||||||
|
if (file.type !== 'application/x-yaml') {
|
||||||
|
setShowError(true);
|
||||||
|
setLabel("Invalid type, only YAML files are accepted.")
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let reader = new FileReader();
|
||||||
|
reader.onload = function(e) {
|
||||||
|
let contents = e.target.result;
|
||||||
|
try {
|
||||||
|
let document = parseDocument(contents);
|
||||||
|
if (document.errors.length > 0) {
|
||||||
|
// not a valid yaml file
|
||||||
|
setShowError(true);
|
||||||
|
setLabel("Invalid file. Make sure your Orchestration is a valid YAML file with a 'steps' section in it.")
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
setShowError(false);
|
||||||
|
setLabel("File loaded successfully.")
|
||||||
|
}
|
||||||
|
setYamlFile(document);
|
||||||
|
} catch (e) {
|
||||||
|
console.error(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reader.readAsText(file);
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<div style={{width:'100%', border:'dashed', textAlign:'center', borderWidth:'1px', padding:'20px'}}>
|
||||||
|
|
||||||
|
<input name="file" type="file" accept=".yaml" onChange={openYAMLFile} />
|
||||||
|
<Typography style={{marginTop:'20px' }} variant="body1" color={showError ? 'error' : ''} >
|
||||||
|
{label}
|
||||||
|
</Typography>
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function ModuleCheckbox({ module, toggleModule, enabledModules, configValues }: { module: object, toggleModule: any, enabledModules: any, configValues: any }) {
|
||||||
|
let name = module.name;
|
||||||
|
const [helpOpen, setHelpOpen] = useState(false);
|
||||||
|
const [configOpen, setConfigOpen] = useState(false);
|
||||||
|
if (name == 'metadata_enricher') {
|
||||||
|
console.log("hi");
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Card>
|
||||||
|
<CardContent>
|
||||||
|
<FormControlLabel
|
||||||
|
control={<Checkbox id={name} onClick={toggleModule} checked={enabledModules.includes(name)} />}
|
||||||
|
label={module.display_name} />
|
||||||
|
</CardContent>
|
||||||
|
<CardActions>
|
||||||
|
<Button size="small" onClick={() => setHelpOpen(true)}>Help</Button>
|
||||||
|
{enabledModules.includes(name) && module.configs && name != 'cli_feeder' ? (
|
||||||
|
<Button size="small" onClick={() => setConfigOpen(true)}>Configure</Button>
|
||||||
|
) : null}
|
||||||
|
</CardActions>
|
||||||
|
</Card>
|
||||||
|
<Dialog
|
||||||
|
open={helpOpen}
|
||||||
|
onClose={() => setHelpOpen(false)}
|
||||||
|
maxWidth="lg"
|
||||||
|
>
|
||||||
|
<DialogTitle>
|
||||||
|
{module.display_name}
|
||||||
|
</DialogTitle>
|
||||||
|
<DialogContent>
|
||||||
|
<ReactMarkdown>
|
||||||
|
{module.manifest.description.split("\n").map((line: string) => line.trim()).join("\n")}
|
||||||
|
</ReactMarkdown>
|
||||||
|
</DialogContent>
|
||||||
|
</Dialog>
|
||||||
|
{module.configs && name != 'cli_feeder' && <ConfigPanel module={module} open={configOpen} setOpen={setConfigOpen} configValues={configValues[module.name]} />}
|
||||||
|
</>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function ConfigPanel({ module, open, setOpen, configValues }: { module: any, open: boolean, setOpen: any, configValues: any }) {
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Dialog
|
||||||
|
key={module}
|
||||||
|
open={open}
|
||||||
|
onClose={() => setOpen(false)}
|
||||||
|
maxWidth="lg"
|
||||||
|
>
|
||||||
|
<DialogTitle>
|
||||||
|
{module.display_name}
|
||||||
|
</DialogTitle>
|
||||||
|
<DialogContent>
|
||||||
|
<Stack key={module} direction="column" spacing={1}>
|
||||||
|
{Object.keys(module.configs).map((config_value: any) => {
|
||||||
|
let config_args = module.configs[config_value];
|
||||||
|
let config_name = config_value.replace(/_/g," ");
|
||||||
|
return (
|
||||||
|
<Box key={config_value}>
|
||||||
|
<FormControl size="small">
|
||||||
|
{ config_args.type === 'bool' ?
|
||||||
|
<FormControlLabel style={{ textTransform: 'capitalize'}} control={<Checkbox checked={configValues[config_value]} size="small" id={`${module}.${config_value}`} />} label={config_name} />
|
||||||
|
:
|
||||||
|
( config_args.type === 'int' ?
|
||||||
|
<TextField size="small" id={`${module}.${config_value}`} label={config_name.capitalize()} value={configValues[config_value]} type="number" />
|
||||||
|
:
|
||||||
|
(
|
||||||
|
config_args.choices !== undefined ?
|
||||||
|
<>
|
||||||
|
<InputLabel>{config_name}</InputLabel>
|
||||||
|
<Select size="small" id={`${module}.${config_value}`}
|
||||||
|
defaultValue={config_args.default} value={configValues[config_value] || ''}>
|
||||||
|
{config_args.choices.map((choice: any) => {
|
||||||
|
return (
|
||||||
|
<MenuItem key={`${module}.${config_value}.${choice}`}
|
||||||
|
value={choice} selected={config_args.default === choice}>{choice}</MenuItem>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</Select>
|
||||||
|
</>
|
||||||
|
:
|
||||||
|
<TextField size="small" id={`${module}.${config_value}`} value={configValues[config_value] || ''} label={config_name.capitalize()} />
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
<FormHelperText style={{ textTransform: 'capitalize'}}>{config_args.help}</FormHelperText>
|
||||||
|
</FormControl>
|
||||||
|
</Box>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</Stack>
|
||||||
|
</DialogContent>
|
||||||
|
</Dialog>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function ModuleTypes({ stepType, toggleModule, enabledModules, configValues }: { stepType: string, toggleModule: any, enabledModules: any, configValues: any }) {
|
||||||
|
const [showError, setShowError] = useState(false);
|
||||||
|
|
||||||
|
const _toggleModule = (event: any) => {
|
||||||
|
// make sure that 'feeder' and 'formatter' types only have one value
|
||||||
|
let name = event.target.id;
|
||||||
|
if (stepType === 'feeder' || stepType === 'formatter') {
|
||||||
|
let checked = event.target.checked;
|
||||||
|
// check how many modules of this type are enabled
|
||||||
|
let modules = steps[stepType].filter((m: string) => (m !== name && enabledModules.includes(m)) || (checked && m === name));
|
||||||
|
if (modules.length > 1) {
|
||||||
|
setShowError(true);
|
||||||
|
} else {
|
||||||
|
setShowError(false);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
setShowError(false);
|
||||||
|
}
|
||||||
|
toggleModule(event);
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Typography id={stepType} variant="h6" style={{ textTransform: 'capitalize' }} >
|
||||||
|
{stepType}s
|
||||||
|
</Typography>
|
||||||
|
{showError ? <Typography variant="body1" color="error" >Only one {stepType} can be enabled at a time.</Typography> : null}
|
||||||
|
<Grid container spacing={1} key={stepType}>
|
||||||
|
{steps[stepType].map((name: string) => {
|
||||||
|
let m = modules[name];
|
||||||
|
return (
|
||||||
|
<Grid key={name} size={{ xs: 6, sm: 4, md: 3 }}>
|
||||||
|
<ModuleCheckbox key={name} module={m} toggleModule={_toggleModule} enabledModules={enabledModules} configValues={configValues} />
|
||||||
|
</Grid>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</Grid>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export default function App() {
|
||||||
|
const [yamlFile, setYamlFile] = useState<Document>(new Document());
|
||||||
|
const [enabledModules, setEnabledModules] = useState<[]>([]);
|
||||||
|
const [configValues, setConfigValues] = useState<{}>(
|
||||||
|
Object.keys(modules).reduce((acc, module) => {
|
||||||
|
acc[module] = {};
|
||||||
|
return acc;
|
||||||
|
}, {})
|
||||||
|
);
|
||||||
|
|
||||||
|
const saveSettings = function(copy: boolean = false) {
|
||||||
|
// edit the yamlFile
|
||||||
|
|
||||||
|
// generate the steps config
|
||||||
|
let stepsConfig = {}
|
||||||
|
module_types.forEach((stepType: string) => {
|
||||||
|
stepsConfig[stepType] = enabledModules.filter((m: string) => steps[stepType].includes(m));
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
// create a yaml file from
|
||||||
|
const finalYaml = {
|
||||||
|
'steps': stepsConfig
|
||||||
|
};
|
||||||
|
|
||||||
|
Object.keys(configValues).map((module: string) => {
|
||||||
|
let module_values = configValues[module];
|
||||||
|
if (module_values) {
|
||||||
|
finalYaml[module] = module_values;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let newFile = new Document(finalYaml);
|
||||||
|
if (copy) {
|
||||||
|
navigator.clipboard.writeText(String(newFile)).then(() => {
|
||||||
|
alert("Settings copied to clipboard.");
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// offer the file for download
|
||||||
|
const blob = new Blob([String(newFile)], { type: 'application/x-yaml' });
|
||||||
|
const url = URL.createObjectURL(blob);
|
||||||
|
const a = document.createElement('a');
|
||||||
|
a.href = url;
|
||||||
|
a.download = 'orchestration.yaml';
|
||||||
|
a.click();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const toggleModule = function (event: any) {
|
||||||
|
let module = event.target.id;
|
||||||
|
let checked = event.target.checked
|
||||||
|
|
||||||
|
if (checked) {
|
||||||
|
setEnabledModules([...enabledModules, module]);
|
||||||
|
} else {
|
||||||
|
setEnabledModules(enabledModules.filter((m: string) => m !== module));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
// load the configs, and set the default values if they exist
|
||||||
|
let newConfigValues = {};
|
||||||
|
Object.keys(modules).map((module: string) => {
|
||||||
|
let m = modules[module];
|
||||||
|
let configs = m.configs;
|
||||||
|
if (!configs) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
newConfigValues[module] = {};
|
||||||
|
Object.keys(configs).map((config: string) => {
|
||||||
|
let config_args = configs[config];
|
||||||
|
if (config_args.default !== undefined) {
|
||||||
|
newConfigValues[module][config] = config_args.default;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
})
|
||||||
|
setConfigValues(newConfigValues);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!yamlFile || yamlFile.contents == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let settings = yamlFile.toJS();
|
||||||
|
// make a deep copy of settings
|
||||||
|
let newEnabledModules = Object.keys(settings['steps']).map((step: string) => {
|
||||||
|
return settings['steps'][step];
|
||||||
|
}).flat();
|
||||||
|
newEnabledModules = newEnabledModules.filter((m: string, i: number) => newEnabledModules.indexOf(m) === i);
|
||||||
|
setEnabledModules(newEnabledModules);
|
||||||
|
}, [yamlFile]);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Container maxWidth="lg">
|
||||||
|
<Box sx={{ my: 4 }}>
|
||||||
|
<Typography variant="h2" >
|
||||||
|
Auto Archiver Settings
|
||||||
|
</Typography>
|
||||||
|
<Box sx={{ my: 4 }}>
|
||||||
|
<Typography variant="h5" >
|
||||||
|
1. Select your <pre style={{display:'inline'}}>orchestration.yaml</pre> settings file.
|
||||||
|
</Typography>
|
||||||
|
<FileDrop setYamlFile={setYamlFile}/>
|
||||||
|
</Box>
|
||||||
|
<Box sx={{ my: 4 }}>
|
||||||
|
<Typography variant="h5" >
|
||||||
|
2. Choose the Modules you wish to enable/disable
|
||||||
|
</Typography>
|
||||||
|
{Object.keys(steps).map((stepType: string) => {
|
||||||
|
return (
|
||||||
|
<ModuleTypes key={stepType} stepType={stepType} toggleModule={toggleModule} enabledModules={enabledModules} configValues={configValues} />
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</Box>
|
||||||
|
<Box sx={{ my: 4 }}>
|
||||||
|
<Typography variant="h5" >
|
||||||
|
3. Configure your Enabled Modules
|
||||||
|
</Typography>
|
||||||
|
<Typography variant="body1" >
|
||||||
|
Next to each module you've enabled, you can click 'Configure' to set the module's settings.
|
||||||
|
</Typography>
|
||||||
|
</Box>
|
||||||
|
<Box sx={{ my: 4 }}>
|
||||||
|
<Typography variant="h5" >
|
||||||
|
4. Save your settings
|
||||||
|
</Typography>
|
||||||
|
<Button variant="contained" color="primary" onClick={() => saveSettings(true)}>Copy Settings to Clipboard</Button>
|
||||||
|
<Button variant="contained" color="primary" onClick={() => saveSettings()}>Save Settings to File</Button>
|
||||||
|
</Box>
|
||||||
|
</Box>
|
||||||
|
</Container>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
import * as React from 'react';
|
||||||
|
import Link from '@mui/material/Link';
|
||||||
|
import SvgIcon, { SvgIconProps } from '@mui/material/SvgIcon';
|
||||||
|
import Typography from '@mui/material/Typography';
|
||||||
|
|
||||||
|
function LightBulbIcon(props: SvgIconProps) {
|
||||||
|
return (
|
||||||
|
<SvgIcon {...props}>
|
||||||
|
<path d="M9 21c0 .55.45 1 1 1h4c.55 0 1-.45 1-1v-1H9v1zm3-19C8.14 2 5 5.14 5 9c0 2.38 1.19 4.47 3 5.74V17c0 .55.45 1 1 1h6c.55 0 1-.45 1-1v-2.26c1.81-1.27 3-3.36 3-5.74 0-3.86-3.14-7-7-7zm2.85 11.1l-.85.6V16h-4v-2.3l-.85-.6C7.8 12.16 7 10.63 7 9c0-2.76 2.24-5 5-5s5 2.24 5 5c0 1.63-.8 3.16-2.15 4.1z" />
|
||||||
|
</SvgIcon>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function ProTip() {
|
||||||
|
return (
|
||||||
|
<Typography sx={{ mt: 6, mb: 3, color: 'text.secondary' }}>
|
||||||
|
<LightBulbIcon sx={{ mr: 1, verticalAlign: 'middle' }} />
|
||||||
|
{'Pro tip: See more '}
|
||||||
|
<Link href="https://mui.com/material-ui/getting-started/templates/">templates</Link>
|
||||||
|
{' in the Material UI documentation.'}
|
||||||
|
</Typography>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
import * as React from 'react';
|
||||||
|
import * as ReactDOM from 'react-dom/client';
|
||||||
|
import { ThemeProvider } from '@mui/material/styles';
|
||||||
|
import { CssBaseline } from '@mui/material';
|
||||||
|
import theme from './theme';
|
||||||
|
import App from './App';
|
||||||
|
|
||||||
|
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||||
|
<React.StrictMode>
|
||||||
|
<ThemeProvider theme={theme}>
|
||||||
|
<CssBaseline />
|
||||||
|
<App />
|
||||||
|
</ThemeProvider>
|
||||||
|
</React.StrictMode>,
|
||||||
|
);
|
||||||
Plik diff jest za duży
Load Diff
|
|
@ -0,0 +1,20 @@
|
||||||
|
import { createTheme } from '@mui/material/styles';
|
||||||
|
import { red } from '@mui/material/colors';
|
||||||
|
|
||||||
|
// A custom theme for this app
|
||||||
|
const theme = createTheme({
|
||||||
|
cssVariables: true,
|
||||||
|
palette: {
|
||||||
|
primary: {
|
||||||
|
main: '#556cd6',
|
||||||
|
},
|
||||||
|
secondary: {
|
||||||
|
main: '#19857b',
|
||||||
|
},
|
||||||
|
error: {
|
||||||
|
main: red.A400,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
export default theme;
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
/// <reference types="vite/client" />
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ESNext",
|
||||||
|
"useDefineForClassFields": true,
|
||||||
|
"lib": ["DOM", "DOM.Iterable", "ESNext"],
|
||||||
|
"allowJs": false,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"esModuleInterop": false,
|
||||||
|
"allowSyntheticDefaultImports": true,
|
||||||
|
"strict": true,
|
||||||
|
"forceConsistentCasingInFileNames": true,
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "Node",
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"isolatedModules": true,
|
||||||
|
"noEmit": true,
|
||||||
|
"jsx": "react-jsx"
|
||||||
|
},
|
||||||
|
"include": ["src"],
|
||||||
|
"references": [{ "path": "./tsconfig.node.json" }]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"composite": true,
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "Node",
|
||||||
|
"allowSyntheticDefaultImports": true
|
||||||
|
},
|
||||||
|
"include": ["vite.config.ts"]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
import { defineConfig } from 'vite';
|
||||||
|
import react from '@vitejs/plugin-react';
|
||||||
|
import { viteSingleFile } from "vite-plugin-singlefile"
|
||||||
|
|
||||||
|
// https://vite.dev/config/
|
||||||
|
export default defineConfig({
|
||||||
|
plugins: [react(), viteSingleFile()],
|
||||||
|
});
|
||||||
|
|
@ -6,7 +6,7 @@
|
||||||
},
|
},
|
||||||
'entry_point': 'csv_db::CSVDb',
|
'entry_point': 'csv_db::CSVDb',
|
||||||
"configs": {
|
"configs": {
|
||||||
"csv_file": {"default": "db.csv", "help": "CSV file name"}
|
"csv_file": {"default": "db.csv", "help": "CSV file name to save metadata to"},
|
||||||
},
|
},
|
||||||
"description": """
|
"description": """
|
||||||
Handles exporting archival results to a CSV file.
|
Handles exporting archival results to a CSV file.
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,9 @@
|
||||||
"bin": [""]
|
"bin": [""]
|
||||||
},
|
},
|
||||||
"configs": {
|
"configs": {
|
||||||
"detect_thumbnails": {"default": True, "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'"}
|
"detect_thumbnails": {"default": True,
|
||||||
|
"help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'",
|
||||||
|
"type": "bool"},
|
||||||
},
|
},
|
||||||
"description": """ """,
|
"description": """ """,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,9 @@
|
||||||
},
|
},
|
||||||
'entry_point': 'ssl_enricher::SSLEnricher',
|
'entry_point': 'ssl_enricher::SSLEnricher',
|
||||||
"configs": {
|
"configs": {
|
||||||
"skip_when_nothing_archived": {"default": True, "help": "if true, will skip enriching when no media is archived"},
|
"skip_when_nothing_archived": {"default": True,
|
||||||
|
"type": 'bool',
|
||||||
|
"help": "if true, will skip enriching when no media is archived"},
|
||||||
},
|
},
|
||||||
"description": """
|
"description": """
|
||||||
Retrieves SSL certificate information for a domain and stores it as a file.
|
Retrieves SSL certificate information for a domain and stores it as a file.
|
||||||
|
|
|
||||||
|
|
@ -17,11 +17,17 @@
|
||||||
"configs": {
|
"configs": {
|
||||||
"profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."},
|
"profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."},
|
||||||
"docker_commands": {"default": None, "help":"if a custom docker invocation is needed"},
|
"docker_commands": {"default": None, "help":"if a custom docker invocation is needed"},
|
||||||
"timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"},
|
"timeout": {"default": 120, "help": "timeout for WACZ generation in seconds", "type": "int"},
|
||||||
"extract_media": {"default": False, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."},
|
"extract_media": {"default": False,
|
||||||
"extract_screenshot": {"default": True, "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."},
|
"type": 'bool',
|
||||||
|
"help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."
|
||||||
|
},
|
||||||
|
"extract_screenshot": {"default": True,
|
||||||
|
"type": 'bool',
|
||||||
|
"help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."
|
||||||
|
},
|
||||||
"socks_proxy_host": {"default": None, "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host"},
|
"socks_proxy_host": {"default": None, "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host"},
|
||||||
"socks_proxy_port": {"default": None, "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"},
|
"socks_proxy_port": {"default": None, "type":"int", "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"},
|
||||||
"proxy_server": {"default": None, "help": "SOCKS server proxy URL, in development"},
|
"proxy_server": {"default": None, "help": "SOCKS server proxy URL, in development"},
|
||||||
},
|
},
|
||||||
"description": """
|
"description": """
|
||||||
|
|
|
||||||
Ładowanie…
Reference in New Issue