mirror
/
auto-archiver
kopia lustrzana https://github.com/bellingcat/auto-archiver
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Remove unused files, set up for RTD

pull/217/head
Patrick Robertson 2025-02-27 21:23:38 +00:00
rodzic 15da907e81
commit 1141c00e9a
12 zmienionych plików z 655 dodań i 283 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

4
.readthedocs.yaml
Wyświetl plik

@ -9,6 +9,7 @@ build:
os: ubuntu-22.04 os: ubuntu-22.04
tools: tools:
python: "3.10" python: "3.10"
node: "22"
jobs: jobs:
post_install: post_install:
- pip install poetry - pip install poetry
@ -17,6 +18,9 @@ build:
# See https://github.com/readthedocs/readthedocs.org/pull/11152/ # See https://github.com/readthedocs/readthedocs.org/pull/11152/
- VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with docs - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with docs
# install node dependencies and build the settings
- cd scripts/settings && npm install && npm run build && cp dist/index.html ../../docs/source/installation/settings_base.html && cd ../..
sphinx: sphinx:
configuration: docs/source/conf.py configuration: docs/source/conf.py

5
docs/source/installation/config_editor.md 100644
Wyświetl plik

@ -0,0 +1,5 @@
# Configuration Editor
```{raw} html
:file: settings.html
```

372
docs/source/installation/settings.html 100644
Wyświetl plik

File diff suppressed because one or more lines are too long

1
docs/source/installation/setup.md
Wyświetl plik

@ -6,6 +6,7 @@
installation.md installation.md
configurations.md configurations.md
config_editor.md
authentication.md authentication.md
requirements.md requirements.md
config_cheatsheet.md config_cheatsheet.md

4
scripts/generate_settings_schema.py
Wyświetl plik

@ -23,7 +23,7 @@ for module in available_modules:
all_modules_ordered_by_type = sorted(available_modules, key=lambda x: (MODULE_TYPES.index(x.type[0]), not x.requires_setup)) all_modules_ordered_by_type = sorted(available_modules, key=lambda x: (MODULE_TYPES.index(x.type[0]), not x.requires_setup))
output_schame = { output_schema = {
'modules': dict((module.name, 'modules': dict((module.name,
{ {
'name': module.name, 'name': module.name,
@ -40,4 +40,4 @@ output_schame = {
current_file_dir = os.path.dirname(os.path.abspath(__file__)) current_file_dir = os.path.dirname(os.path.abspath(__file__))
output_file = os.path.join(current_file_dir, 'settings/src/schema.json') output_file = os.path.join(current_file_dir, 'settings/src/schema.json')
with open(output_file, 'w') as file: with open(output_file, 'w') as file:
json.dump(output_schame, file, indent=4, cls=SchemaEncoder) json.dump(output_schema, file, indent=4, cls=SchemaEncoder)

1
scripts/settings/index.html
Wyświetl plik

@ -2,7 +2,6 @@
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="utf-8" /> <meta charset="utf-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="initial-scale=1, width=device-width" /> <meta name="viewport" content="initial-scale=1, width=device-width" />
<!-- Fonts to support Material Design --> <!-- Fonts to support Material Design -->
<link rel="preconnect" href="https://fonts.googleapis.com" /> <link rel="preconnect" href="https://fonts.googleapis.com" />

1
scripts/settings/public/vite.svg
Wyświetl plik

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
Side by Side

Przed

Szerokość:  |  Wysokość:  |  Rozmiar: 1.5 KiB

28
scripts/settings/src/App.tsx
Wyświetl plik

@ -50,7 +50,7 @@ function FileDrop({ setYamlFile }: { setYamlFile: React.Dispatch<React.SetStateA
return; return;
} }
let reader = new FileReader(); let reader = new FileReader();
reader.onload = function(e) { reader.onload = function (e) {
let contents = e.target ? e.target.result : ''; let contents = e.target ? e.target.result : '';
try { try {
let document = parseDocument(contents as string); let document = parseDocument(contents as string);
@ -72,10 +72,10 @@ function FileDrop({ setYamlFile }: { setYamlFile: React.Dispatch<React.SetStateA
} }
return ( return (
<> <>
<div style={{width:'100%', border:'dashed', textAlign:'center', borderWidth:'1px', padding:'20px'}}> <div style={{ width: '100%', border: 'dashed', textAlign: 'center', borderWidth: '1px', padding: '20px' }}>
<input name="file" type="file" accept=".yaml" onChange={openYAMLFile} /> <input name="file" type="file" accept=".yaml" onChange={openYAMLFile} />
<Typography style={{marginTop:'20px' }} variant="body1" color={showError ? 'error' : ''} > <Typography style={{ marginTop: '20px' }} variant="body1" color={showError ? 'error' : ''} >
{label} {label}
</Typography> </Typography>
</div> </div>
@ -97,7 +97,7 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues
// make sure that 'feeder' and 'formatter' types only have one value // make sure that 'feeder' and 'formatter' types only have one value
let name = event.target.id; let name = event.target.id;
let checked = event.target.checked; let checked = event.target.checked;
if (stepType === 'feeder' || stepType === 'formatter') { if (stepType === 'feeders' || stepType === 'formatters') {
// check how many modules of this type are enabled // check how many modules of this type are enabled
const checkedModules = enabledModules[stepType].filter(([m, enabled]: [string, boolean]) => { const checkedModules = enabledModules[stepType].filter(([m, enabled]: [string, boolean]) => {
return (m !== name && enabled) || (checked && m === name) return (m !== name && enabled) || (checked && m === name)
@ -110,7 +110,7 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues
} else { } else {
setShowError(false); setShowError(false);
} }
let newEnabledModules = Object.fromEntries(Object.keys(enabledModules).map((type : string) => { let newEnabledModules = Object.fromEntries(Object.keys(enabledModules).map((type: string) => {
return [type, enabledModules[type].map(([m, enabled]: [string, boolean]) => { return [type, enabledModules[type].map(([m, enabled]: [string, boolean]) => {
return (m === name) ? [m, checked] : [m, enabled]; return (m === name) ? [m, checked] : [m, enabled];
})]; })];
@ -154,10 +154,11 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues
{stepType} {stepType}
</Typography> </Typography>
<Typography variant="body1" > <Typography variant="body1" >
Select the {stepType} you wish to enable. You can drag and drop them to reorder them. Select the {stepType} you wish to enable. You can drag and move to reorder.
Learn more about {stepType} <a href={`https://auto-archiver.readthedocs.io/en/latest/modules/${stepType.slice(0,-1)}.html`} target="_blank">here</a>.
</Typography> </Typography>
</Box> </Box>
{showError ? <Typography variant="body1" color="error" >Only one {stepType} can be enabled at a time.</Typography> : null} {showError ? <Typography variant="body1" color="error" >Only one {stepType.slice(0,-1)} can be enabled at a time.</Typography> : null}
<DndContext <DndContext
sensors={sensors} sensors={sensors}
@ -175,12 +176,12 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues
})} })}
<DragOverlay> <DragOverlay>
{activeId ? ( {activeId ? (
<div <div
style={{ style={{
width: "100%", width: "100%",
height: "100%", height: "100%",
backgroundColor: "grey", backgroundColor: "grey",
opacity:0.1, opacity: 0.1,
}} }}
></div> ></div>
@ -209,7 +210,7 @@ export default function App() {
}, {}) }, {})
); );
const saveSettings = function(copy: boolean = false) { const saveSettings = function (copy: boolean = false) {
// edit the yamlFile // edit the yamlFile
// generate the steps config // generate the steps config
@ -301,14 +302,11 @@ export default function App() {
return ( return (
<Container maxWidth="lg"> <Container maxWidth="lg">
<Box sx={{ my: 4 }}> <Box sx={{ my: 4 }}>
<Typography variant="h2" >
Auto Archiver Settings
</Typography>
<Box sx={{ my: 4 }}> <Box sx={{ my: 4 }}>
<Typography variant="h5" > <Typography variant="h5" >
1. Select your <pre style={{display:'inline'}}>orchestration.yaml</pre> settings file. 1. Select your orchestration.yaml settings file.
</Typography> </Typography>
<FileDrop setYamlFile={setYamlFile}/> <FileDrop setYamlFile={setYamlFile} />
</Box> </Box>
<Box sx={{ my: 4 }}> <Box sx={{ my: 4 }}>
<Typography variant="h5" > <Typography variant="h5" >

23
scripts/settings/src/ProTip.tsx
Wyświetl plik

@ -1,23 +0,0 @@
import * as React from 'react';
import Link from '@mui/material/Link';
import SvgIcon, { SvgIconProps } from '@mui/material/SvgIcon';
import Typography from '@mui/material/Typography';
function LightBulbIcon(props: SvgIconProps) {
return (
<SvgIcon {...props}>
<path d="M9 21c0 .55.45 1 1 1h4c.55 0 1-.45 1-1v-1H9v1zm3-19C8.14 2 5 5.14 5 9c0 2.38 1.19 4.47 3 5.74V17c0 .55.45 1 1 1h6c.55 0 1-.45 1-1v-2.26c1.81-1.27 3-3.36 3-5.74 0-3.86-3.14-7-7-7zm2.85 11.1l-.85.6V16h-4v-2.3l-.85-.6C7.8 12.16 7 10.63 7 9c0-2.76 2.24-5 5-5s5 2.24 5 5c0 1.63-.8 3.16-2.15 4.1z" />
</SvgIcon>
);
}
export default function ProTip() {
return (
<Typography sx={{ mt: 6, mb: 3, color: 'text.secondary' }}>
<LightBulbIcon sx={{ mr: 1, verticalAlign: 'middle' }} />
{'Pro tip: See more '}
<Link href="https://mui.com/material-ui/getting-started/templates/">templates</Link>
{' in the Material UI documentation.'}
</Typography>
);
}

101
scripts/settings/src/StepCard.tsx
Wyświetl plik

@ -23,17 +23,22 @@ import {
TextField, TextField,
Stack, Stack,
Typography, Typography,
} from '@mui/material'; } from '@mui/material';
import Grid from '@mui/material/Grid2'; import Grid from '@mui/material/Grid2';
import DragIndicatorIcon from '@mui/icons-material/DragIndicator'; import DragIndicatorIcon from '@mui/icons-material/DragIndicator';
import { Module } from "./types"; import HelpIconOutlined from '@mui/icons-material/HelpOutline';
import { Module, Config } from "./types";
Object.defineProperty(String.prototype, 'capitalize', {
value: function() { // adds 'capitalize' method to String prototype
declare global {
interface String {
capitalize(): string;
}
}
String.prototype.capitalize = function (this: string) {
return this.charAt(0).toUpperCase() + this.slice(1); return this.charAt(0).toUpperCase() + this.slice(1);
}, };
enumerable: false
});
const StepCard = ({ const StepCard = ({
type, type,
@ -72,25 +77,29 @@ const StepCard = ({
return ( return (
<Grid ref={setNodeRef} size={{ xs: 6, sm: 4, md: 3 }} style={style}> <Grid ref={setNodeRef} size={{ xs: 6, sm: 4, md: 3 }} style={style}>
<Card> <Card >
<CardHeader <CardHeader
title={ title={
<FormControlLabel <FormControlLabel
control={<Checkbox id={name} onClick={toggleModule} checked={enabled} />} style={{paddingRight: '0 !important'}}
control={<Checkbox sx={{paddingTop:0, paddingBottom:0}} id={name} onClick={toggleModule} checked={enabled} />}
label={module.display_name} /> label={module.display_name} />
} }
action ={
<IconButton size="small" {...listeners} {...attributes}>
<DragIndicatorIcon />
</IconButton>
}
/> />
<CardActions> <CardActions>
<Button size="small" onClick={() => setHelpOpen(true)}>Info</Button> <Box sx={{ justifyContent: 'space-between', display: 'flex', width: '100%' }}>
<Box>
<IconButton size="small" onClick={() => setHelpOpen(true)}>
<HelpIconOutlined />
</IconButton>
{enabled && module.configs && name != 'cli_feeder' ? ( {enabled && module.configs && name != 'cli_feeder' ? (
<Button size="small" onClick={() => setConfigOpen(true)}>Configure</Button> <Button size="small" onClick={() => setConfigOpen(true)}>Configure</Button>
) : null} ) : null}
</Box>
<IconButton size="small" sx={{textAlight: 'right', cursor: 'grab' }} {...listeners} {...attributes}>
<DragIndicatorIcon />
</IconButton>
</Box>
</CardActions> </CardActions>
</Card> </Card>
<Dialog <Dialog
@ -110,37 +119,21 @@ const StepCard = ({
{module.configs && name != 'cli_feeder' && <ConfigPanel module={module} open={configOpen} setOpen={setConfigOpen} configValues={configValues} />} {module.configs && name != 'cli_feeder' && <ConfigPanel module={module} open={configOpen} setOpen={setConfigOpen} configValues={configValues} />}
</Grid> </Grid>
) )
} }
function ConfigPanel({ module, open, setOpen, configValues }: { module: any, open: boolean, setOpen: any, configValues: any }) {
function ConfigField({ config_value, module, configValues }: { config_value: any, module: Module, configValues: any }) {
function setConfigValue(config: any, value: any) { function setConfigValue(config: any, value: any) {
configValues[module.name][config] = value; configValues[module.name][config] = value;
} }
return ( const config_args: Config = module.configs[config_value];
<> const config_name: string = config_value.replace(/_/g, " ");
<Dialog
key={module}
open={open}
onClose={() => setOpen(false)}
maxWidth="lg"
>
<DialogTitle>
{module.display_name}
</DialogTitle>
<DialogContent>
<Stack key={module} direction="column" spacing={1}>
{Object.keys(module.configs).map((config_value: any) => {
const config_args = module.configs[config_value];
const config_name = config_value.replace(/_/g," ");
const config_display_name = config_name.capitalize(); const config_display_name = config_name.capitalize();
const value = configValues[module.name][config_value] || config_args.default; const value = configValues[module.name][config_value] || config_args.default;
return ( return (
<Box key={config_value}> <Box>
<Typography variant='body1' style={{fontWeight : 'bold'}}>{config_display_name}</Typography> <Typography variant='body1' style={{ fontWeight: 'bold' }}>{config_display_name} {config_args.required && (`(required)`)} </Typography>
<FormControl size="small"> <FormControl size="small">
{ config_args.type === 'bool' ? {config_args.type === 'bool' ?
<FormControlLabel control={ <FormControlLabel control={
<Checkbox defaultChecked={value} size="small" id={`${module}.${config_value}`} <Checkbox defaultChecked={value} size="small" id={`${module}.${config_value}`}
onChange={(e) => { onChange={(e) => {
@ -165,7 +158,7 @@ function ConfigPanel({ module, open, setOpen, configValues }: { module: any, ope
})} })}
</Select> </Select>
: :
( config_args.type === 'json_loader' ? (config_args.type === 'json_loader' ?
<TextField multiline size="small" id={`${module}.${config_value}`} defaultValue={JSON.stringify(value, null, 2)} rows={6} onChange={ <TextField multiline size="small" id={`${module}.${config_value}`} defaultValue={JSON.stringify(value, null, 2)} rows={6} onChange={
(e) => { (e) => {
try { try {
@ -180,15 +173,37 @@ function ConfigPanel({ module, open, setOpen, configValues }: { module: any, ope
<TextField size="small" id={`${module}.${config_value}`} defaultValue={value} type={config_args.type === 'int' ? 'number' : 'text'} <TextField size="small" id={`${module}.${config_value}`} defaultValue={value} type={config_args.type === 'int' ? 'number' : 'text'}
onChange={(e) => { onChange={(e) => {
setConfigValue(config_value, e.target.value); setConfigValue(config_value, e.target.value);
}} /> }}
required={config_args.required}
/>
) )
) )
} }
{config_args.type !== 'bool' && ( {config_args.type !== 'bool' && (
<FormHelperText >{config_args.help}</FormHelperText> <FormHelperText >{config_args.help.capitalize()}</FormHelperText>
)} )}
</FormControl> </FormControl>
</Box> </Box>
)
}
function ConfigPanel({ module, open, setOpen, configValues }: { module: Module, open: boolean, setOpen: any, configValues: any }) {
return (
<>
<Dialog
open={open}
onClose={() => setOpen(false)}
maxWidth="lg"
>
<DialogTitle>
{module.display_name}
</DialogTitle>
<DialogContent>
<Stack direction="column" spacing={1}>
{Object.keys(module.configs).map((config_value: any) => {
return (
<ConfigField key={config_value} config_value={config_value} module={module} configValues={configValues} />
); );
})} })}
</Stack> </Stack>
@ -196,6 +211,6 @@ function ConfigPanel({ module, open, setOpen, configValues }: { module: any, ope
</Dialog> </Dialog>
</> </>
); );
} }
export default StepCard; export default StepCard;

3
scripts/settings/src/types.d.ts vendored
Wyświetl plik

@ -3,6 +3,9 @@ export interface Config {
description: string; description: string;
type: string?; type: string?;
default: any; default: any;
help: string;
choices: string[];
required: boolean;
} }
interface Manifest { interface Manifest {

1
scripts/settings/src/vite-env.d.ts vendored
Wyświetl plik

@ -1 +0,0 @@
/// <reference types="vite/client" />