diff --git a/.gitignore b/.gitignore index f31bc6c..72d8531 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,5 @@ docs/_build/ docs/source/autoapi/ docs/source/modules/autogen/ scripts/settings_page.html +scripts/settings/src/schema.json .vite diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 6dc9fe5..8d32a3a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -21,7 +21,7 @@ build: # generate the config editor page. Schema then HTML - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry run python scripts/generate_settings_schema.py # install node dependencies and build the settings - - cd scripts/settings && npm install && npm run build && yes | cp dist/index.html ../../docs/source/installation/settings_base.html && cd ../.. + - cd scripts/settings && npm install && npm run build && yes | cp -v dist/index.html ../../docs/source/installation/settings.html && cd ../.. sphinx: diff --git a/README.md b/README.md index 8baa722..b8dd530 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ View the [Installation Guide](https://auto-archiver.readthedocs.io/en/latest/ins To get started quickly using Docker: -`docker pull bellingcat/auto-archiver && docker run --rm -v secrets:/app/secrets bellingcat/auto-archiver --config secrets/orchestration.yaml` +`docker pull bellingcat/auto-archiver && docker run -it --rm -v secrets:/app/secrets bellingcat/auto-archiver --config secrets/orchestration.yaml` Or pip: diff --git a/docs/source/development/docs.md b/docs/source/development/docs.md index bb389ff..9e9bf0a 100644 --- a/docs/source/development/docs.md +++ b/docs/source/development/docs.md @@ -36,3 +36,12 @@ open docs/_build/html/index.html sphinx-autobuild docs/source docs/_build/html ``` + +### Managing Readthedocs (RTD) Versions + +Version management is done at [https://app.readthedocs.org/projects/auto-archiver/](https://app.readthedocs.org/projects/auto-archiver/) +(login required). Once logged in, you can create new versions, delete old versions or change visibility of versions. More info on +[RTD](https://docs.readthedocs.com/platform/stable/versions.html). + +Currently, the Auto Archiver project is set up to automatically create a new docs version for each `vX.Y.Z` release. For more on this, +see the RTD [instructions on automation](https://docs.readthedocs.com/platform/stable/guides/automation-rules.html) or edit the existing automation rule in the project settings. \ No newline at end of file diff --git a/docs/source/how_to/gsheets_setup.md b/docs/source/how_to/gsheets_setup.md index ade8024..af17274 100644 --- a/docs/source/how_to/gsheets_setup.md +++ b/docs/source/how_to/gsheets_setup.md @@ -86,7 +86,7 @@ gsheet_feeder_db: You can also pass these settings directly on the command line without having to edit the file, here'a an example of how to do that (using docker): -`docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --gsheet_feeder_db.sheet "My Awesome Sheet 2"`. +`docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --gsheet_feeder_db.sheet "My Awesome Sheet 2"`. Here, the sheet name has been overridden/specified in the command line invocation. diff --git a/docs/source/installation/faq.md b/docs/source/installation/faq.md new file mode 100644 index 0000000..246fbc4 --- /dev/null +++ b/docs/source/installation/faq.md @@ -0,0 +1,60 @@ +# Frequently Asked Questions + + +### Q: What websites does the Auto Archiver support? +**A:** The Auto Archiver works for a large variety of sites. Firstly, the Auto Archiver can download +and archive any video website supported by YT-DLP, a powerful video-downloading tool ([full list of of +sites here](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md)). Aside from these sites, +there are various different 'Extractors' for specific websites. See the full list of extractors that +are available on the [extractors](../modules/extractor.md) page. Some sites supported include: + +* Twitter +* Instagram +* Telegram +* VKontact +* Tiktok +* Bluesky + +```{note} What websites the Auto Archiver can archie depends on what extractors you have enabled in +your configuration. See [configuration](./configurations.md) for more info. +``` + +### Q: Does the Auto Archiver only work for social media posts ? +**A:** No, the Auto Archiver can archive any web page on the internet, not just social media posts. +However, for social media posts Auto Archiver can extract more relevant/useful information (such as +post comments, likes, author etc.) which may not be available for a generic website. If you are looking +to more generally archive webpages, then you should make sure to enable the [](../modules/autogen/extractor/wacz_extractor_enricher.md) +and the [](../modules/autogen/extractor/wayback_extractor_enricher.md). + +### Q: What kind of data is stored for each webpage that's archived? +**A:** This depends on the website archived, but more generally, for social media posts any videos and photos in +the post will be archived. For video sites, the video will be downloaded separately. For most of these sites, additional +metadata such as published date, uploader/author and ratings/comments will also be saved. Additionally, further data can be +saved depending on the enrichers that you have enabled. Some other types of data saved are timestamps if you have the +[](../modules/autogen/enricher/timestamping_enricher.md) or [](../modules/autogen/enricher/opentimestamps_enricher.md) enabled, +screenshots of the web page with the [](../modules/autogen/enricher/screenshot_enricher.md), and for videos, thumbnails of the +video with the [](../modules/autogen/enricher/thumbnail_enricher.md). You can also store things like hashes (SHA256, or pdq hashes) +with the various hash enrichers. + +### Q: Where is my data stored? +**A:** With the default configuration, data is stored on your local computer in the `local_storage` folder. You can adjust these settings by +changing the [storage modules](../modules/storage.md) you have enabled. For example, you could choose to store your data in an S3 bucket or +on Google Drive. + +```{note} +You can choose to store your data in multiple places, for example your local drive **and** an S3 bucket for redundancy. +``` + +### Q: What should I do is something doesn't work? +**A:** First, read through the log files to see if you can find a specific reason why something isn't working. Learn more about logging +and how to enable debug logging in the [Logging Howto](../how_to/logging.md). + +If you cannot find an answer in the logs, then try searching this documentation or existing / closed issues on the [Github Issue Tracker](https://github.com/bellingcat/auto-archiver/issues?q=is%3Aissue%20). If you still cannot find an answer, then consider opening an issue on the Github Issue Tracker or asking in the Bellingcat Discord +'Auto Archiver' group. + +#### Common reasons why an archiving might not work: + +* The website may have temporarily adjusted its settings - sometimes sites like Telegram or Twitter adjust their scraping protection settings. Often, +waiting a day or two and then trying again can work. +* The site requires you to be logged in - you could try using cookies or authentication to bypass any blocks. See [](../installation/authentication.md) for more information. +* The website you're trying to archive has changed its settings/structure. Make sure you're using the latest version of Auto Archiver and try again. diff --git a/docs/source/installation/installation.md b/docs/source/installation/installation.md index eff0720..40b21cf 100644 --- a/docs/source/installation/installation.md +++ b/docs/source/installation/installation.md @@ -1,5 +1,11 @@ # Installation +```{toctree} +:maxdepth: 1 + +upgrading.md +``` + There are 3 main ways to use the auto-archiver. We recommend the 'docker' method for most uses. This installs all the requirements in one command. 1. Easiest (recommended): [via docker](#installing-with-docker) diff --git a/docs/source/installation/settings.html b/docs/source/installation/settings.html index 915ee8d..cd9f0ec 100644 --- a/docs/source/installation/settings.html +++ b/docs/source/installation/settings.html @@ -1,68 +1,4 @@ - +`;n[d+2]==="\\"&&n[d+3]==="n"&&n[d+4]!=='"';)u+=` +`,d+=2;u+=l,n[d+2]===" "&&(u+="\\"),d+=1,f=d+1}break;default:d+=1}return u=f?u+n.slice(f):n,i?u:Lf(u,l,zc,zf(t,!1))}function gm(e,t){if(t.options.singleQuote===!1||t.implicitKey&&e.includes(` +`)||/[ \t]\n|\n[ \t]/.test(e))return xl(e,t);const n=t.indent||(If(e)?" ":""),i="'"+e.replace(/'/g,"''").replace(/\n+/g,`$& +${n}`)+"'";return t.implicitKey?i:Lf(i,n,Gx,zf(t,!1))}function Oo(e,t){const{singleQuote:n}=t.options;let i;if(n===!1)i=xl;else{const o=e.includes('"'),l=e.includes("'");o&&!l?i=gm:l&&!o?i=xl:i=n?gm:xl}return i(e,t)}let ym;try{ym=new RegExp(`(^|(? +`;let g,v;for(v=n.length;v>0;--v){const R=n[v-1];if(R!==` +`&&R!==" "&&R!==" ")break}let b=n.substring(v);const w=b.indexOf(` +`);w===-1?g="-":n===b||w!==b.length-1?(g="+",l&&l()):g="",b&&(n=n.slice(0,-b.length),b[b.length-1]===` +`&&(b=b.slice(0,-1)),b=b.replace(ym,`$&${p}`));let S=!1,C,A=-1;for(C=0;C{D=!0});const z=Lf(`${O}${R}${b}`,p,mm,P);if(!D)return`>${M} +${p}${z}`}return n=n.replace(/\n+/g,`$&${p}`),`|${M} +${p}${O}${n}${b}`}function zL(e,t,n,i){const{type:o,value:l}=e,{actualString:u,implicitKey:f,indent:d,indentStep:p,inFlow:m}=t;if(f&&l.includes(` +`)||m&&/[[\]{},]/.test(l))return Oo(l,t);if(!l||/^[\n\t ,[\]{}#&*!|>'"%@`]|^[?-]$|^[?-][ \t]|[\n:][ \t]|[ \t]\n|[\n\t ]#|[\n\t :]$/.test(l))return f||m||!l.includes(` +`)?Oo(l,t):Ic(e,t,n,i);if(!f&&!m&&o!==Ie.PLAIN&&l.includes(` +`))return Ic(e,t,n,i);if(If(l)){if(d==="")return t.forceBlockIndent=!0,Ic(e,t,n,i);if(f&&d===p)return Oo(l,t)}const g=l.replace(/\n+/g,`$& +${d}`);if(u){const v=S=>{var C;return S.default&&S.tag!=="tag:yaml.org,2002:str"&&((C=S.test)==null?void 0:C.test(g))},{compat:b,tags:w}=t.doc.schema;if(w.some(v)||b!=null&&b.some(v))return Oo(l,t)}return f?g:Lf(g,d,Gx,zf(t,!1))}function Sg(e,t,n,i){const{implicitKey:o,inFlow:l}=t,u=typeof e.value=="string"?e:Object.assign({},e,{value:String(e.value)});let{type:f}=e;f!==Ie.QUOTE_DOUBLE&&/[\x00-\x08\x0b-\x1f\x7f-\x9f\u{D800}-\u{DFFF}]/u.test(u.value)&&(f=Ie.QUOTE_DOUBLE);const d=m=>{switch(m){case Ie.BLOCK_FOLDED:case Ie.BLOCK_LITERAL:return o||l?Oo(u.value,t):Ic(u,t,n,i);case Ie.QUOTE_DOUBLE:return xl(u.value,t);case Ie.QUOTE_SINGLE:return gm(u.value,t);case Ie.PLAIN:return zL(u,t,n,i);default:return null}};let p=d(f);if(p===null){const{defaultKeyType:m,defaultStringType:g}=t.options,v=o&&m||g;if(p=d(v),p===null)throw new Error(`Unsupported default string type ${v}`)}return p}function Kx(e,t){const n=Object.assign({blockQuote:!0,commentString:DL,defaultKeyType:null,defaultStringType:"PLAIN",directives:null,doubleQuotedAsJSON:!1,doubleQuotedMinMultiLineLength:40,falseStr:"false",flowCollectionPadding:!0,indentSeq:!0,lineWidth:80,minContentWidth:20,nullStr:"null",simpleKeys:!1,singleQuote:null,trueStr:"true",verifyAliasOrder:!0},e.schema.toStringOptions,t);let i;switch(n.collectionStyle){case"block":i=!1;break;case"flow":i=!0;break;default:i=null}return{anchors:new Set,doc:e,flowCollectionPadding:n.flowCollectionPadding?" ":"",indent:"",indentStep:typeof n.indent=="number"?" ".repeat(n.indent):" ",inFlow:i,options:n}}function IL(e,t){var o;if(t.tag){const l=e.filter(u=>u.tag===t.tag);if(l.length>0)return l.find(u=>u.format===t.format)??l[0]}let n,i;if(Tt(t)){i=t.value;let l=e.filter(u=>{var f;return(f=u.identify)==null?void 0:f.call(u,i)});if(l.length>1){const u=l.filter(f=>f.test);u.length>0&&(l=u)}n=l.find(u=>u.format===t.format)??l.find(u=>!u.format)}else i=t,n=e.find(l=>l.nodeClass&&i instanceof l.nodeClass);if(!n){const l=((o=i==null?void 0:i.constructor)==null?void 0:o.name)??typeof i;throw new Error(`Tag not resolved for ${l} value`)}return n}function BL(e,t,{anchors:n,doc:i}){if(!i.directives)return"";const o=[],l=(Tt(e)||Ut(e))&&e.anchor;l&&Ux(l)&&(n.add(l),o.push(`&${l}`));const u=e.tag?e.tag:t.default?null:t.tag;return u&&o.push(i.directives.tagString(u)),o.join(" ")}function Ho(e,t,n,i){var d;if(qt(e))return e.toString(t,n,i);if(Qo(e)){if(t.doc.directives)return e.toString(t);if((d=t.resolvedAliases)!=null&&d.has(e))throw new TypeError("Cannot stringify circular structure without alias nodes");t.resolvedAliases?t.resolvedAliases.add(e):t.resolvedAliases=new Set([e]),e=e.resolve(t.doc)}let o;const l=Ht(e)?e:t.doc.createNode(e,{onTagObj:p=>o=p});o||(o=IL(t.doc.schema.tags,l));const u=BL(l,o,t);u.length>0&&(t.indentAtStart=(t.indentAtStart??0)+u.length+1);const f=typeof o.stringify=="function"?o.stringify(l,t,n,i):Tt(l)?Sg(l,t,n,i):l.toString(t,n,i);return u?Tt(l)||f[0]==="{"||f[0]==="["?`${u} ${f}`:`${u} +${t.indent}${f}`:f}function jL({key:e,value:t},n,i,o){const{allNullValues:l,doc:u,indent:f,indentStep:d,options:{commentString:p,indentSeq:m,simpleKeys:g}}=n;let v=Ht(e)&&e.comment||null;if(g){if(v)throw new Error("With simple keys, key nodes cannot have comments");if(Ut(e)||!Ht(e)&&typeof e=="object"){const P="With simple keys, collection cannot be used as a key value";throw new Error(P)}}let b=!g&&(!e||v&&t==null&&!n.inFlow||Ut(e)||(Tt(e)?e.type===Ie.BLOCK_FOLDED||e.type===Ie.BLOCK_LITERAL:typeof e=="object"));n=Object.assign({},n,{allNullValues:!1,implicitKey:!b&&(g||!l),indent:f+d});let w=!1,S=!1,C=Ho(e,n,()=>w=!0,()=>S=!0);if(!b&&!n.inFlow&&C.length>1024){if(g)throw new Error("With simple keys, single line scalar must not span more than 1024 characters");b=!0}if(n.inFlow){if(l||t==null)return w&&i&&i(),C===""?"?":b?`? ${C}`:C}else if(l&&!g||t==null&&b)return C=`? ${C}`,v&&!w?C+=Ta(C,n.indent,p(v)):S&&o&&o(),C;w&&(v=null),b?(v&&(C+=Ta(C,n.indent,p(v))),C=`? ${C} +${f}:`):(C=`${C}:`,v&&(C+=Ta(C,n.indent,p(v))));let A,O,_;Ht(t)?(A=!!t.spaceBefore,O=t.commentBefore,_=t.comment):(A=!1,O=null,_=null,t&&typeof t=="object"&&(t=u.createNode(t))),n.implicitKey=!1,!b&&!v&&Tt(t)&&(n.indentAtStart=C.length+1),S=!1,!m&&d.length>=2&&!n.inFlow&&!b&&Zl(t)&&!t.flow&&!t.tag&&!t.anchor&&(n.indent=n.indent.substring(2));let M=!1;const R=Ho(t,n,()=>M=!0,()=>S=!0);let D=" ";if(v||A||O){if(D=A?` +`:"",O){const P=p(O);D+=` +${pi(P,n.indent)}`}R===""&&!n.inFlow?D===` +`&&(D=` + +`):D+=` +${n.indent}`}else if(!b&&Ut(t)){const P=R[0],z=R.indexOf(` +`),$=z!==-1,E=n.inFlow??t.flow??t.items.length===0;if($||!E){let I=!1;if($&&(P==="&"||P==="!")){let U=R.indexOf(" ");P==="&"&&U!==-1&&Ue===vc||typeof e=="symbol"&&e.description===vc,default:"key",tag:"tag:yaml.org,2002:merge",test:/^<<$/,resolve:()=>Object.assign(new Ie(Symbol(vc)),{addToJSMap:Yx}),stringify:()=>vc},PL=(e,t)=>(hi.identify(t)||Tt(t)&&(!t.type||t.type===Ie.PLAIN)&&hi.identify(t.value))&&(e==null?void 0:e.doc.schema.tags.some(n=>n.tag===hi.tag&&n.default));function Yx(e,t,n){if(n=e&&Qo(n)?n.resolve(e.doc):n,Zl(n))for(const i of n.items)xh(e,t,i);else if(Array.isArray(n))for(const i of n)xh(e,t,i);else xh(e,t,n)}function xh(e,t,n){const i=e&&Qo(n)?n.resolve(e.doc):n;if(!Ql(i))throw new Error("Merge sources must be maps or map aliases");const o=i.toJSON(null,e,Map);for(const[l,u]of o)t instanceof Map?t.has(l)||t.set(l,u):t instanceof Set?t.add(l):Object.prototype.hasOwnProperty.call(t,l)||Object.defineProperty(t,l,{value:u,writable:!0,enumerable:!0,configurable:!0});return t}function Xx(e,t,{key:n,value:i}){if(Ht(n)&&n.addToJSMap)n.addToJSMap(e,t,i);else if(PL(e,n))Yx(e,t,i);else{const o=pr(n,"",e);if(t instanceof Map)t.set(o,pr(i,o,e));else if(t instanceof Set)t.add(o);else{const l=UL(n,o,e),u=pr(i,l,e);l in t?Object.defineProperty(t,l,{value:u,writable:!0,enumerable:!0,configurable:!0}):t[l]=u}}return t}function UL(e,t,n){if(t===null)return"";if(typeof t!="object")return String(t);if(Ht(e)&&(n!=null&&n.doc)){const i=Kx(n.doc,{});i.anchors=new Set;for(const l of n.anchors.keys())i.anchors.add(l.anchor);i.inFlow=!0,i.inStringifyKey=!0;const o=e.toString(i);if(!n.mapKeyWarned){let l=JSON.stringify(o);l.length>40&&(l=l.substring(0,36)+'..."'),$L(n.doc.options.logLevel,`Keys with collection values will be stringified due to JS Object restrictions: ${l}. Set mapAsMap: true to use object keys.`),n.mapKeyWarned=!0}return o}return JSON.stringify(t)}function wg(e,t,n){const i=Ll(e,void 0,n),o=Ll(t,void 0,n);return new kn(i,o)}class kn{constructor(t,n=null){Object.defineProperty(this,hr,{value:Px}),this.key=t,this.value=n}clone(t){let{key:n,value:i}=this;return Ht(n)&&(n=n.clone(t)),Ht(i)&&(i=i.clone(t)),new kn(n,i)}toJSON(t,n){const i=n!=null&&n.mapAsMap?new Map:{};return Xx(n,i,this)}toString(t,n,i){return t!=null&&t.doc?jL(this,t,n,i):JSON.stringify(this)}}function Wx(e,t,n){return(t.inFlow??e.flow?qL:HL)(e,t,n)}function HL({comment:e,items:t},n,{blockItemPrefix:i,flowChars:o,itemIndent:l,onChompKeep:u,onComment:f}){const{indent:d,options:{commentString:p}}=n,m=Object.assign({},n,{indent:l,type:null});let g=!1;const v=[];for(let w=0;wC=null,()=>g=!0);C&&(A+=Ta(A,l,p(C))),g&&C&&(g=!1),v.push(i+A)}let b;if(v.length===0)b=o.start+o.end;else{b=v[0];for(let w=1;wC=null);wm||A.includes(` +`))&&(p=!0),g.push(A),m=g.length}const{start:v,end:b}=n;if(g.length===0)return v+b;if(!p){const w=g.reduce((S,C)=>S+C.length+2,2);p=t.options.lineWidth>0&&w>t.options.lineWidth}if(p){let w=v;for(const S of g)w+=S?` +${l}${o}${S}`:` +`;return`${w} +${o}${b}`}else return`${v}${u}${g.join(" ")}${u}${b}`}function ef({indent:e,options:{commentString:t}},n,i,o){if(i&&o&&(i=i.replace(/^\n+/,"")),i){const l=pi(t(i),e);n.push(l.trimStart())}}function Aa(e,t){const n=Tt(t)?t.value:t;for(const i of e)if(qt(i)&&(i.key===t||i.key===n||Tt(i.key)&&i.key.value===n))return i}class dr extends Vx{static get tagName(){return"tag:yaml.org,2002:map"}constructor(t){super(Gi,t),this.items=[]}static from(t,n,i){const{keepUndefined:o,replacer:l}=i,u=new this(t),f=(d,p)=>{if(typeof l=="function")p=l.call(n,d,p);else if(Array.isArray(l)&&!l.includes(d))return;(p!==void 0||o)&&u.items.push(wg(d,p,i))};if(n instanceof Map)for(const[d,p]of n)f(d,p);else if(n&&typeof n=="object")for(const d of Object.keys(n))f(d,n[d]);return typeof t.sortMapEntries=="function"&&u.items.sort(t.sortMapEntries),u}add(t,n){var u;let i;qt(t)?i=t:!t||typeof t!="object"||!("key"in t)?i=new kn(t,t==null?void 0:t.value):i=new kn(t.key,t.value);const o=Aa(this.items,i.key),l=(u=this.schema)==null?void 0:u.sortMapEntries;if(o){if(!n)throw new Error(`Key ${i.key} already set`);Tt(o.value)&&Fx(i.value)?o.value.value=i.value:o.value=i.value}else if(l){const f=this.items.findIndex(d=>l(i,d)<0);f===-1?this.items.push(i):this.items.splice(f,0,i)}else this.items.push(i)}delete(t){const n=Aa(this.items,t);return n?this.items.splice(this.items.indexOf(n),1).length>0:!1}get(t,n){const i=Aa(this.items,t),o=i==null?void 0:i.value;return(!n&&Tt(o)?o.value:o)??void 0}has(t){return!!Aa(this.items,t)}set(t,n){this.add(new kn(t,n),!0)}toJSON(t,n,i){const o=i?new i:n!=null&&n.mapAsMap?new Map:{};n!=null&&n.onCreate&&n.onCreate(o);for(const l of this.items)Xx(n,o,l);return o}toString(t,n,i){if(!t)return JSON.stringify(this);for(const o of this.items)if(!qt(o))throw new Error(`Map items must all be pairs; found ${JSON.stringify(o)} instead`);return!t.allNullValues&&this.hasAllNullValues(!1)&&(t=Object.assign({},t,{allNullValues:!0})),Wx(this,t,{blockItemPrefix:"",flowChars:{start:"{",end:"}"},itemIndent:t.indent||"",onChompKeep:i,onComment:n})}}const Zo={collection:"map",default:!0,nodeClass:dr,tag:"tag:yaml.org,2002:map",resolve(e,t){return Ql(e)||t("Expected a mapping for this tag"),e},createNode:(e,t,n)=>dr.from(e,t,n)};class Oa extends Vx{static get tagName(){return"tag:yaml.org,2002:seq"}constructor(t){super(Wo,t),this.items=[]}add(t){this.items.push(t)}delete(t){const n=Sc(t);return typeof n!="number"?!1:this.items.splice(n,1).length>0}get(t,n){const i=Sc(t);if(typeof i!="number")return;const o=this.items[i];return!n&&Tt(o)?o.value:o}has(t){const n=Sc(t);return typeof n=="number"&&n=0?t:null}const Jo={collection:"seq",default:!0,nodeClass:Oa,tag:"tag:yaml.org,2002:seq",resolve(e,t){return Zl(e)||t("Expected a sequence for this tag"),e},createNode:(e,t,n)=>Oa.from(e,t,n)},Bf={identify:e=>typeof e=="string",default:!0,tag:"tag:yaml.org,2002:str",resolve:e=>e,stringify(e,t,n,i){return t=Object.assign({actualString:!0},t),Sg(e,t,n,i)}},jf={identify:e=>e==null,createNode:()=>new Ie(null),default:!0,tag:"tag:yaml.org,2002:null",test:/^(?:~|[Nn]ull|NULL)?$/,resolve:()=>new Ie(null),stringify:({source:e},t)=>typeof e=="string"&&jf.test.test(e)?e:t.options.nullStr},xg={identify:e=>typeof e=="boolean",default:!0,tag:"tag:yaml.org,2002:bool",test:/^(?:[Tt]rue|TRUE|[Ff]alse|FALSE)$/,resolve:e=>new Ie(e[0]==="t"||e[0]==="T"),stringify({source:e,value:t},n){if(e&&xg.test.test(e)){const i=e[0]==="t"||e[0]==="T";if(t===i)return e}return t?n.options.trueStr:n.options.falseStr}};function Tr({format:e,minFractionDigits:t,tag:n,value:i}){if(typeof i=="bigint")return String(i);const o=typeof i=="number"?i:Number(i);if(!isFinite(o))return isNaN(o)?".nan":o<0?"-.inf":".inf";let l=JSON.stringify(i);if(!e&&t&&(!n||n==="tag:yaml.org,2002:float")&&/^\d/.test(l)){let u=l.indexOf(".");u<0&&(u=l.length,l+=".");let f=t-(l.length-u-1);for(;f-- >0;)l+="0"}return l}const Qx={identify:e=>typeof e=="number",default:!0,tag:"tag:yaml.org,2002:float",test:/^(?:[-+]?\.(?:inf|Inf|INF)|\.nan|\.NaN|\.NAN)$/,resolve:e=>e.slice(-3).toLowerCase()==="nan"?NaN:e[0]==="-"?Number.NEGATIVE_INFINITY:Number.POSITIVE_INFINITY,stringify:Tr},Zx={identify:e=>typeof e=="number",default:!0,tag:"tag:yaml.org,2002:float",format:"EXP",test:/^[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)[eE][-+]?[0-9]+$/,resolve:e=>parseFloat(e),stringify(e){const t=Number(e.value);return isFinite(t)?t.toExponential():Tr(e)}},Jx={identify:e=>typeof e=="number",default:!0,tag:"tag:yaml.org,2002:float",test:/^[-+]?(?:\.[0-9]+|[0-9]+\.[0-9]*)$/,resolve(e){const t=new Ie(parseFloat(e)),n=e.indexOf(".");return n!==-1&&e[e.length-1]==="0"&&(t.minFractionDigits=e.length-n-1),t},stringify:Tr},$f=e=>typeof e=="bigint"||Number.isInteger(e),Cg=(e,t,n,{intAsBigInt:i})=>i?BigInt(e):parseInt(e.substring(t),n);function eC(e,t,n){const{value:i}=e;return $f(i)&&i>=0?n+i.toString(t):Tr(e)}const tC={identify:e=>$f(e)&&e>=0,default:!0,tag:"tag:yaml.org,2002:int",format:"OCT",test:/^0o[0-7]+$/,resolve:(e,t,n)=>Cg(e,2,8,n),stringify:e=>eC(e,8,"0o")},nC={identify:$f,default:!0,tag:"tag:yaml.org,2002:int",test:/^[-+]?[0-9]+$/,resolve:(e,t,n)=>Cg(e,0,10,n),stringify:Tr},rC={identify:e=>$f(e)&&e>=0,default:!0,tag:"tag:yaml.org,2002:int",format:"HEX",test:/^0x[0-9a-fA-F]+$/,resolve:(e,t,n)=>Cg(e,2,16,n),stringify:e=>eC(e,16,"0x")},FL=[Zo,Jo,Bf,jf,xg,tC,nC,rC,Qx,Zx,Jx];function n1(e){return typeof e=="bigint"||Number.isInteger(e)}const wc=({value:e})=>JSON.stringify(e),VL=[{identify:e=>typeof e=="string",default:!0,tag:"tag:yaml.org,2002:str",resolve:e=>e,stringify:wc},{identify:e=>e==null,createNode:()=>new Ie(null),default:!0,tag:"tag:yaml.org,2002:null",test:/^null$/,resolve:()=>null,stringify:wc},{identify:e=>typeof e=="boolean",default:!0,tag:"tag:yaml.org,2002:bool",test:/^true$|^false$/,resolve:e=>e==="true",stringify:wc},{identify:n1,default:!0,tag:"tag:yaml.org,2002:int",test:/^-?(?:0|[1-9][0-9]*)$/,resolve:(e,t,{intAsBigInt:n})=>n?BigInt(e):parseInt(e,10),stringify:({value:e})=>n1(e)?e.toString():JSON.stringify(e)},{identify:e=>typeof e=="number",default:!0,tag:"tag:yaml.org,2002:float",test:/^-?(?:0|[1-9][0-9]*)(?:\.[0-9]*)?(?:[eE][-+]?[0-9]+)?$/,resolve:e=>parseFloat(e),stringify:wc}],GL={default:!0,tag:"",test:/^/,resolve(e,t){return t(`Unresolved plain scalar ${JSON.stringify(e)}`),e}},KL=[Zo,Jo].concat(VL,GL),kg={identify:e=>e instanceof Uint8Array,default:!1,tag:"tag:yaml.org,2002:binary",resolve(e,t){if(typeof atob=="function"){const n=atob(e.replace(/[\n\r]/g,"")),i=new Uint8Array(n.length);for(let o=0;o1&&t("Each pair must have its own sequence indicator");const o=i.items[0]||new kn(new Ie(null));if(i.commentBefore&&(o.key.commentBefore=o.key.commentBefore?`${i.commentBefore} +${o.key.commentBefore}`:i.commentBefore),i.comment){const l=o.value??o.key;l.comment=l.comment?`${i.comment} +${l.comment}`:i.comment}i=o}e.items[n]=qt(i)?i:new kn(i)}}else t("Expected a sequence for this tag");return e}function aC(e,t,n){const{replacer:i}=n,o=new Oa(e);o.tag="tag:yaml.org,2002:pairs";let l=0;if(t&&Symbol.iterator in Object(t))for(let u of t){typeof i=="function"&&(u=i.call(t,String(l++),u));let f,d;if(Array.isArray(u))if(u.length===2)f=u[0],d=u[1];else throw new TypeError(`Expected [key, value] tuple: ${u}`);else if(u&&u instanceof Object){const p=Object.keys(u);if(p.length===1)f=p[0],d=u[f];else throw new TypeError(`Expected tuple with one key, not ${p.length} keys`)}else f=u;o.items.push(wg(f,d,n))}return o}const Eg={collection:"seq",default:!1,tag:"tag:yaml.org,2002:pairs",resolve:iC,createNode:aC};class Io extends Oa{constructor(){super(),this.add=dr.prototype.add.bind(this),this.delete=dr.prototype.delete.bind(this),this.get=dr.prototype.get.bind(this),this.has=dr.prototype.has.bind(this),this.set=dr.prototype.set.bind(this),this.tag=Io.tag}toJSON(t,n){if(!n)return super.toJSON(t);const i=new Map;n!=null&&n.onCreate&&n.onCreate(i);for(const o of this.items){let l,u;if(qt(o)?(l=pr(o.key,"",n),u=pr(o.value,l,n)):l=pr(o,"",n),i.has(l))throw new Error("Ordered maps must not include duplicate keys");i.set(l,u)}return i}static from(t,n,i){const o=aC(t,n,i),l=new this;return l.items=o.items,l}}Io.tag="tag:yaml.org,2002:omap";const Tg={collection:"seq",identify:e=>e instanceof Map,nodeClass:Io,default:!1,tag:"tag:yaml.org,2002:omap",resolve(e,t){const n=iC(e,t),i=[];for(const{key:o}of n.items)Tt(o)&&(i.includes(o.value)?t(`Ordered maps must not include duplicate keys: ${o.value}`):i.push(o.value));return Object.assign(new Io,n)},createNode:(e,t,n)=>Io.from(e,t,n)};function oC({value:e,source:t},n){return t&&(e?sC:lC).test.test(t)?t:e?n.options.trueStr:n.options.falseStr}const sC={identify:e=>e===!0,default:!0,tag:"tag:yaml.org,2002:bool",test:/^(?:Y|y|[Yy]es|YES|[Tt]rue|TRUE|[Oo]n|ON)$/,resolve:()=>new Ie(!0),stringify:oC},lC={identify:e=>e===!1,default:!0,tag:"tag:yaml.org,2002:bool",test:/^(?:N|n|[Nn]o|NO|[Ff]alse|FALSE|[Oo]ff|OFF)$/,resolve:()=>new Ie(!1),stringify:oC},YL={identify:e=>typeof e=="number",default:!0,tag:"tag:yaml.org,2002:float",test:/^(?:[-+]?\.(?:inf|Inf|INF)|\.nan|\.NaN|\.NAN)$/,resolve:e=>e.slice(-3).toLowerCase()==="nan"?NaN:e[0]==="-"?Number.NEGATIVE_INFINITY:Number.POSITIVE_INFINITY,stringify:Tr},XL={identify:e=>typeof e=="number",default:!0,tag:"tag:yaml.org,2002:float",format:"EXP",test:/^[-+]?(?:[0-9][0-9_]*)?(?:\.[0-9_]*)?[eE][-+]?[0-9]+$/,resolve:e=>parseFloat(e.replace(/_/g,"")),stringify(e){const t=Number(e.value);return isFinite(t)?t.toExponential():Tr(e)}},WL={identify:e=>typeof e=="number",default:!0,tag:"tag:yaml.org,2002:float",test:/^[-+]?(?:[0-9][0-9_]*)?\.[0-9_]*$/,resolve(e){const t=new Ie(parseFloat(e.replace(/_/g,""))),n=e.indexOf(".");if(n!==-1){const i=e.substring(n+1).replace(/_/g,"");i[i.length-1]==="0"&&(t.minFractionDigits=i.length)}return t},stringify:Tr},Jl=e=>typeof e=="bigint"||Number.isInteger(e);function Pf(e,t,n,{intAsBigInt:i}){const o=e[0];if((o==="-"||o==="+")&&(t+=1),e=e.substring(t).replace(/_/g,""),i){switch(n){case 2:e=`0b${e}`;break;case 8:e=`0o${e}`;break;case 16:e=`0x${e}`;break}const u=BigInt(e);return o==="-"?BigInt(-1)*u:u}const l=parseInt(e,n);return o==="-"?-1*l:l}function Ag(e,t,n){const{value:i}=e;if(Jl(i)){const o=i.toString(t);return i<0?"-"+n+o.substr(1):n+o}return Tr(e)}const QL={identify:Jl,default:!0,tag:"tag:yaml.org,2002:int",format:"BIN",test:/^[-+]?0b[0-1_]+$/,resolve:(e,t,n)=>Pf(e,2,2,n),stringify:e=>Ag(e,2,"0b")},ZL={identify:Jl,default:!0,tag:"tag:yaml.org,2002:int",format:"OCT",test:/^[-+]?0[0-7_]+$/,resolve:(e,t,n)=>Pf(e,1,8,n),stringify:e=>Ag(e,8,"0")},JL={identify:Jl,default:!0,tag:"tag:yaml.org,2002:int",test:/^[-+]?[0-9][0-9_]*$/,resolve:(e,t,n)=>Pf(e,0,10,n),stringify:Tr},ez={identify:Jl,default:!0,tag:"tag:yaml.org,2002:int",format:"HEX",test:/^[-+]?0x[0-9a-fA-F_]+$/,resolve:(e,t,n)=>Pf(e,2,16,n),stringify:e=>Ag(e,16,"0x")};class Bo extends dr{constructor(t){super(t),this.tag=Bo.tag}add(t){let n;qt(t)?n=t:t&&typeof t=="object"&&"key"in t&&"value"in t&&t.value===null?n=new kn(t.key,null):n=new kn(t,null),Aa(this.items,n.key)||this.items.push(n)}get(t,n){const i=Aa(this.items,t);return!n&&qt(i)?Tt(i.key)?i.key.value:i.key:i}set(t,n){if(typeof n!="boolean")throw new Error(`Expected boolean value for set(key, value) in a YAML set, not ${typeof n}`);const i=Aa(this.items,t);i&&!n?this.items.splice(this.items.indexOf(i),1):!i&&n&&this.items.push(new kn(t))}toJSON(t,n){return super.toJSON(t,n,Set)}toString(t,n,i){if(!t)return JSON.stringify(this);if(this.hasAllNullValues(!0))return super.toString(Object.assign({},t,{allNullValues:!0}),n,i);throw new Error("Set items must all have null values")}static from(t,n,i){const{replacer:o}=i,l=new this(t);if(n&&Symbol.iterator in Object(n))for(let u of n)typeof o=="function"&&(u=o.call(n,u,u)),l.items.push(wg(u,null,i));return l}}Bo.tag="tag:yaml.org,2002:set";const _g={collection:"map",identify:e=>e instanceof Set,nodeClass:Bo,default:!1,tag:"tag:yaml.org,2002:set",createNode:(e,t,n)=>Bo.from(e,t,n),resolve(e,t){if(Ql(e)){if(e.hasAllNullValues(!0))return Object.assign(new Bo,e);t("Set items must all have null values")}else t("Expected a mapping for this tag");return e}};function Rg(e,t){const n=e[0],i=n==="-"||n==="+"?e.substring(1):e,o=u=>t?BigInt(u):Number(u),l=i.replace(/_/g,"").split(":").reduce((u,f)=>u*o(60)+o(f),o(0));return n==="-"?o(-1)*l:l}function uC(e){let{value:t}=e,n=u=>u;if(typeof t=="bigint")n=u=>BigInt(u);else if(isNaN(t)||!isFinite(t))return Tr(e);let i="";t<0&&(i="-",t*=n(-1));const o=n(60),l=[t%o];return t<60?l.unshift(0):(t=(t-l[0])/o,l.unshift(t%o),t>=60&&(t=(t-l[0])/o,l.unshift(t))),i+l.map(u=>String(u).padStart(2,"0")).join(":").replace(/000000\d*$/,"")}const cC={identify:e=>typeof e=="bigint"||Number.isInteger(e),default:!0,tag:"tag:yaml.org,2002:int",format:"TIME",test:/^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+$/,resolve:(e,t,{intAsBigInt:n})=>Rg(e,n),stringify:uC},fC={identify:e=>typeof e=="number",default:!0,tag:"tag:yaml.org,2002:float",format:"TIME",test:/^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*$/,resolve:e=>Rg(e,!1),stringify:uC},Uf={identify:e=>e instanceof Date,default:!0,tag:"tag:yaml.org,2002:timestamp",test:RegExp("^([0-9]{4})-([0-9]{1,2})-([0-9]{1,2})(?:(?:t|T|[ \\t]+)([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2}(\\.[0-9]+)?)(?:[ \\t]*(Z|[-+][012]?[0-9](?::[0-9]{2})?))?)?$"),resolve(e){const t=e.match(Uf.test);if(!t)throw new Error("!!timestamp expects a date, starting with yyyy-mm-dd");const[,n,i,o,l,u,f]=t.map(Number),d=t[7]?Number((t[7]+"00").substr(1,3)):0;let p=Date.UTC(n,i-1,o,l||0,u||0,f||0,d);const m=t[8];if(m&&m!=="Z"){let g=Rg(m,!1);Math.abs(g)<30&&(g*=60),p-=6e4*g}return new Date(p)},stringify:({value:e})=>e.toISOString().replace(/(T00:00:00)?\.000Z$/,"")},r1=[Zo,Jo,Bf,jf,sC,lC,QL,ZL,JL,ez,YL,XL,WL,kg,hi,Tg,Eg,_g,cC,fC,Uf],i1=new Map([["core",FL],["failsafe",[Zo,Jo,Bf]],["json",KL],["yaml11",r1],["yaml-1.1",r1]]),a1={binary:kg,bool:xg,float:Jx,floatExp:Zx,floatNaN:Qx,floatTime:fC,int:nC,intHex:rC,intOct:tC,intTime:cC,map:Zo,merge:hi,null:jf,omap:Tg,pairs:Eg,seq:Jo,set:_g,timestamp:Uf},tz={"tag:yaml.org,2002:binary":kg,"tag:yaml.org,2002:merge":hi,"tag:yaml.org,2002:omap":Tg,"tag:yaml.org,2002:pairs":Eg,"tag:yaml.org,2002:set":_g,"tag:yaml.org,2002:timestamp":Uf};function Ch(e,t,n){const i=i1.get(t);if(i&&!e)return n&&!i.includes(hi)?i.concat(hi):i.slice();let o=i;if(!o)if(Array.isArray(e))o=[];else{const l=Array.from(i1.keys()).filter(u=>u!=="yaml11").map(u=>JSON.stringify(u)).join(", ");throw new Error(`Unknown schema "${t}"; use one of ${l} or define customTags array`)}if(Array.isArray(e))for(const l of e)o=o.concat(l);else typeof e=="function"&&(o=e(o.slice()));return n&&(o=o.concat(hi)),o.reduce((l,u)=>{const f=typeof u=="string"?a1[u]:u;if(!f){const d=JSON.stringify(u),p=Object.keys(a1).map(m=>JSON.stringify(m)).join(", ");throw new Error(`Unknown custom tag ${d}; use one of ${p}`)}return l.includes(f)||l.push(f),l},[])}const nz=(e,t)=>e.keyt.key?1:0;let rz=class dC{constructor({compat:t,customTags:n,merge:i,resolveKnownTags:o,schema:l,sortMapEntries:u,toStringDefaults:f}){this.compat=Array.isArray(t)?Ch(t,"compat"):t?Ch(null,t):null,this.name=typeof l=="string"&&l||"core",this.knownTags=o?tz:{},this.tags=Ch(n,this.name,i),this.toStringOptions=f??null,Object.defineProperty(this,Gi,{value:Zo}),Object.defineProperty(this,Vr,{value:Bf}),Object.defineProperty(this,Wo,{value:Jo}),this.sortMapEntries=typeof u=="function"?u:u===!0?nz:null}clone(){const t=Object.create(dC.prototype,Object.getOwnPropertyDescriptors(this));return t.tags=this.tags.slice(),t}};function iz(e,t){var d;const n=[];let i=t.directives===!0;if(t.directives!==!1&&e.directives){const p=e.directives.toString(e);p?(n.push(p),i=!0):e.directives.docStart&&(i=!0)}i&&n.push("---");const o=Kx(e,t),{commentString:l}=o.options;if(e.commentBefore){n.length!==1&&n.unshift("");const p=l(e.commentBefore);n.unshift(pi(p,""))}let u=!1,f=null;if(e.contents){if(Ht(e.contents)){if(e.contents.spaceBefore&&i&&n.push(""),e.contents.commentBefore){const g=l(e.contents.commentBefore);n.push(pi(g,""))}o.forceBlockIndent=!!e.comment,f=e.contents.comment}const p=f?void 0:()=>u=!0;let m=Ho(e.contents,o,()=>f=null,p);f&&(m+=Ta(m,"",l(f))),(m[0]==="|"||m[0]===">")&&n[n.length-1]==="---"?n[n.length-1]=`--- ${m}`:n.push(m)}else n.push(Ho(e.contents,o));if((d=e.directives)!=null&&d.docEnd)if(e.comment){const p=l(e.comment);p.includes(` +`)?(n.push("..."),n.push(pi(p,""))):n.push(`... ${p}`)}else n.push("...");else{let p=e.comment;p&&u&&(p=p.replace(/^\n+/,"")),p&&((!u||f)&&n[n.length-1]!==""&&n.push(""),n.push(pi(l(p),"")))}return n.join(` +`)+` +`}class eu{constructor(t,n,i){this.commentBefore=null,this.comment=null,this.errors=[],this.warnings=[],Object.defineProperty(this,hr,{value:hm});let o=null;typeof n=="function"||Array.isArray(n)?o=n:i===void 0&&n&&(i=n,n=void 0);const l=Object.assign({intAsBigInt:!1,keepSourceTokens:!1,logLevel:"warn",prettyErrors:!0,strict:!0,stringKeys:!1,uniqueKeys:!0,version:"1.2"},i);this.options=l;let{version:u}=l;i!=null&&i._directives?(this.directives=i._directives.atDocument(),this.directives.yaml.explicit&&(u=this.directives.yaml.version)):this.directives=new Cn({version:u}),this.setSchema(u,i),this.contents=t===void 0?null:this.createNode(t,o,i)}clone(){const t=Object.create(eu.prototype,{[hr]:{value:hm}});return t.commentBefore=this.commentBefore,t.comment=this.comment,t.errors=this.errors.slice(),t.warnings=this.warnings.slice(),t.options=Object.assign({},this.options),this.directives&&(t.directives=this.directives.clone()),t.schema=this.schema.clone(),t.contents=Ht(this.contents)?this.contents.clone(t.schema):this.contents,this.range&&(t.range=this.range.slice()),t}add(t){Co(this.contents)&&this.contents.add(t)}addIn(t,n){Co(this.contents)&&this.contents.addIn(t,n)}createAlias(t,n){if(!t.anchor){const i=Hx(this);t.anchor=!n||i.has(n)?qx(n||"a",i):n}return new vg(t.anchor)}createNode(t,n,i){let o;if(typeof n=="function")t=n.call({"":t},"",t),o=n;else if(Array.isArray(n)){const C=O=>typeof O=="number"||O instanceof String||O instanceof Number,A=n.filter(C).map(String);A.length>0&&(n=n.concat(A)),o=n}else i===void 0&&n&&(i=n,n=void 0);const{aliasDuplicateObjects:l,anchorPrefix:u,flow:f,keepUndefined:d,onTagObj:p,tag:m}=i??{},{onAnchor:g,setAnchors:v,sourceObjects:b}=OL(this,u||"a"),w={aliasDuplicateObjects:l??!0,keepUndefined:d??!1,onAnchor:g,onTagObj:p,replacer:o,schema:this.schema,sourceObjects:b},S=Ll(t,m,w);return f&&Ut(S)&&(S.flow=!0),v(),S}createPair(t,n,i={}){const o=this.createNode(t,null,i),l=this.createNode(n,null,i);return new kn(o,l)}delete(t){return Co(this.contents)?this.contents.delete(t):!1}deleteIn(t){return pl(t)?this.contents==null?!1:(this.contents=null,!0):Co(this.contents)?this.contents.deleteIn(t):!1}get(t,n){return Ut(this.contents)?this.contents.get(t,n):void 0}getIn(t,n){return pl(t)?!n&&Tt(this.contents)?this.contents.value:this.contents:Ut(this.contents)?this.contents.getIn(t,n):void 0}has(t){return Ut(this.contents)?this.contents.has(t):!1}hasIn(t){return pl(t)?this.contents!==void 0:Ut(this.contents)?this.contents.hasIn(t):!1}set(t,n){this.contents==null?this.contents=Jc(this.schema,[t],n):Co(this.contents)&&this.contents.set(t,n)}setIn(t,n){pl(t)?this.contents=n:this.contents==null?this.contents=Jc(this.schema,Array.from(t),n):Co(this.contents)&&this.contents.setIn(t,n)}setSchema(t,n={}){typeof t=="number"&&(t=String(t));let i;switch(t){case"1.1":this.directives?this.directives.yaml.version="1.1":this.directives=new Cn({version:"1.1"}),i={resolveKnownTags:!1,schema:"yaml-1.1"};break;case"1.2":case"next":this.directives?this.directives.yaml.version=t:this.directives=new Cn({version:t}),i={resolveKnownTags:!0,schema:"core"};break;case null:this.directives&&delete this.directives,i=null;break;default:{const o=JSON.stringify(t);throw new Error(`Expected '1.1', '1.2' or null as first argument, but found: ${o}`)}}if(n.schema instanceof Object)this.schema=n.schema;else if(i)this.schema=new rz(Object.assign(i,n));else throw new Error("With a null YAML version, the { schema: Schema } option is required")}toJS({json:t,jsonArg:n,mapAsMap:i,maxAliasCount:o,onAnchor:l,reviver:u}={}){const f={anchors:new Map,doc:this,keep:!t,mapAsMap:i===!0,mapKeyWarned:!1,maxAliasCount:typeof o=="number"?o:100},d=pr(this.contents,n??"",f);if(typeof l=="function")for(const{count:p,res:m}of f.anchors.values())l(m,p);return typeof u=="function"?Ro(u,{"":d},"",d):d}toJSON(t,n){return this.toJS({json:!0,jsonArg:t,mapAsMap:!1,onAnchor:n})}toString(t={}){if(this.errors.length>0)throw new Error("Document with errors cannot be stringified");if("indent"in t&&(!Number.isInteger(t.indent)||Number(t.indent)<=0)){const n=JSON.stringify(t.indent);throw new Error(`"indent" option must be a positive integer, not ${n}`)}return iz(this,t)}}function Co(e){if(Ut(e))return!0;throw new Error("Expected a YAML collection as document contents")}class pC extends Error{constructor(t,n,i,o){super(),this.name=t,this.code=i,this.message=o,this.pos=n}}class hl extends pC{constructor(t,n,i){super("YAMLParseError",t,n,i)}}class az extends pC{constructor(t,n,i){super("YAMLWarning",t,n,i)}}const o1=(e,t)=>n=>{if(n.pos[0]===-1)return;n.linePos=n.pos.map(f=>t.linePos(f));const{line:i,col:o}=n.linePos[0];n.message+=` at line ${i}, column ${o}`;let l=o-1,u=e.substring(t.lineStarts[i-1],t.lineStarts[i]).replace(/[\n\r]+$/,"");if(l>=60&&u.length>80){const f=Math.min(l-39,u.length-79);u="…"+u.substring(f),l-=f-1}if(u.length>80&&(u=u.substring(0,79)+"…"),i>1&&/^ *$/.test(u.substring(0,l))){let f=e.substring(t.lineStarts[i-2],t.lineStarts[i-1]);f.length>80&&(f=f.substring(0,79)+`… +`),u=f+u}if(/[^ ]/.test(u)){let f=1;const d=n.linePos[1];d&&d.line===i&&d.col>o&&(f=Math.max(1,Math.min(d.col-o,80-l)));const p=" ".repeat(l)+"^".repeat(f);n.message+=`: + +${u} +${p} +`}};function qo(e,{flow:t,indicator:n,next:i,offset:o,onError:l,parentIndent:u,startOnNewline:f}){let d=!1,p=f,m=f,g="",v="",b=!1,w=!1,S=null,C=null,A=null,O=null,_=null,M=null,R=null;for(const z of e)switch(w&&(z.type!=="space"&&z.type!=="newline"&&z.type!=="comma"&&l(z.offset,"MISSING_CHAR","Tags and anchors must be separated from the next token by white space"),w=!1),S&&(p&&z.type!=="comment"&&z.type!=="newline"&&l(S,"TAB_AS_INDENT","Tabs are not allowed as indentation"),S=null),z.type){case"space":!t&&(n!=="doc-start"||(i==null?void 0:i.type)!=="flow-collection")&&z.source.includes(" ")&&(S=z),m=!0;break;case"comment":{m||l(z,"MISSING_CHAR","Comments must be separated from other tokens by white space characters");const $=z.source.substring(1)||" ";g?g+=v+$:g=$,v="",p=!1;break}case"newline":p?g?g+=z.source:(!M||n!=="seq-item-ind")&&(d=!0):v+=z.source,p=!0,b=!0,(C||A)&&(O=z),m=!0;break;case"anchor":C&&l(z,"MULTIPLE_ANCHORS","A node can have at most one anchor"),z.source.endsWith(":")&&l(z.offset+z.source.length-1,"BAD_ALIAS","Anchor ending in : is ambiguous",!0),C=z,R===null&&(R=z.offset),p=!1,m=!1,w=!0;break;case"tag":{A&&l(z,"MULTIPLE_TAGS","A node can have at most one tag"),A=z,R===null&&(R=z.offset),p=!1,m=!1,w=!0;break}case n:(C||A)&&l(z,"BAD_PROP_ORDER",`Anchors and tags must be after the ${z.source} indicator`),M&&l(z,"UNEXPECTED_TOKEN",`Unexpected ${z.source} in ${t??"collection"}`),M=z,p=n==="seq-item-ind"||n==="explicit-key-ind",m=!1;break;case"comma":if(t){_&&l(z,"UNEXPECTED_TOKEN",`Unexpected , in ${t}`),_=z,p=!1,m=!1;break}default:l(z,"UNEXPECTED_TOKEN",`Unexpected ${z.type} token`),p=!1,m=!1}const D=e[e.length-1],P=D?D.offset+D.source.length:o;return w&&i&&i.type!=="space"&&i.type!=="newline"&&i.type!=="comma"&&(i.type!=="scalar"||i.source!=="")&&l(i.offset,"MISSING_CHAR","Tags and anchors must be separated from the next token by white space"),S&&(p&&S.indent<=u||(i==null?void 0:i.type)==="block-map"||(i==null?void 0:i.type)==="block-seq")&&l(S,"TAB_AS_INDENT","Tabs are not allowed as indentation"),{comma:_,found:M,spaceBefore:d,comment:g,hasNewline:b,anchor:C,tag:A,newlineAfterProp:O,end:P,start:R??P}}function zl(e){if(!e)return null;switch(e.type){case"alias":case"scalar":case"double-quoted-scalar":case"single-quoted-scalar":if(e.source.includes(` +`))return!0;if(e.end){for(const t of e.end)if(t.type==="newline")return!0}return!1;case"flow-collection":for(const t of e.items){for(const n of t.start)if(n.type==="newline")return!0;if(t.sep){for(const n of t.sep)if(n.type==="newline")return!0}if(zl(t.key)||zl(t.value))return!0}return!1;default:return!0}}function bm(e,t,n){if((t==null?void 0:t.type)==="flow-collection"){const i=t.end[0];i.indent===e&&(i.source==="]"||i.source==="}")&&zl(t)&&n(i,"BAD_INDENT","Flow end indicator should be more indented than parent",!0)}}function hC(e,t,n){const{uniqueKeys:i}=e.options;if(i===!1)return!1;const o=typeof i=="function"?i:(l,u)=>l===u||Tt(l)&&Tt(u)&&l.value===u.value;return t.some(l=>o(l.key,n))}const s1="All mapping items must start at the same column";function oz({composeNode:e,composeEmptyNode:t},n,i,o,l){var m;const u=(l==null?void 0:l.nodeClass)??dr,f=new u(n.schema);n.atRoot&&(n.atRoot=!1);let d=i.offset,p=null;for(const g of i.items){const{start:v,key:b,sep:w,value:S}=g,C=qo(v,{indicator:"explicit-key-ind",next:b??(w==null?void 0:w[0]),offset:d,onError:o,parentIndent:i.indent,startOnNewline:!0}),A=!C.found;if(A){if(b&&(b.type==="block-seq"?o(d,"BLOCK_AS_IMPLICIT_KEY","A block sequence may not be used as an implicit map key"):"indent"in b&&b.indent!==i.indent&&o(d,"BAD_INDENT",s1)),!C.anchor&&!C.tag&&!w){p=C.end,C.comment&&(f.comment?f.comment+=` +`+C.comment:f.comment=C.comment);continue}(C.newlineAfterProp||zl(b))&&o(b??v[v.length-1],"MULTILINE_IMPLICIT_KEY","Implicit keys need to be on a single line")}else((m=C.found)==null?void 0:m.indent)!==i.indent&&o(d,"BAD_INDENT",s1);n.atKey=!0;const O=C.end,_=b?e(n,b,C,o):t(n,O,v,null,C,o);n.schema.compat&&bm(i.indent,b,o),n.atKey=!1,hC(n,f.items,_)&&o(O,"DUPLICATE_KEY","Map keys must be unique");const M=qo(w??[],{indicator:"map-value-ind",next:S,offset:_.range[2],onError:o,parentIndent:i.indent,startOnNewline:!b||b.type==="block-scalar"});if(d=M.end,M.found){A&&((S==null?void 0:S.type)==="block-map"&&!M.hasNewline&&o(d,"BLOCK_AS_IMPLICIT_KEY","Nested mappings are not allowed in compact mappings"),n.options.strict&&C.starte&&(e.type==="block-map"||e.type==="block-seq");function lz({composeNode:e,composeEmptyNode:t},n,i,o,l){const u=i.start.source==="{",f=u?"flow map":"flow sequence",d=(l==null?void 0:l.nodeClass)??(u?dr:Oa),p=new d(n.schema);p.flow=!0;const m=n.atRoot;m&&(n.atRoot=!1),n.atKey&&(n.atKey=!1);let g=i.offset+i.start.source.length;for(let C=0;C0){const C=tu(w,S,n.options.strict,o);C.comment&&(p.comment?p.comment+=` +`+C.comment:p.comment=C.comment),p.range=[i.offset,S,C.offset]}else p.range=[i.offset,S,S];return p}function Th(e,t,n,i,o,l){const u=n.type==="block-map"?oz(e,t,n,i,l):n.type==="block-seq"?sz(e,t,n,i,l):lz(e,t,n,i,l),f=u.constructor;return o==="!"||o===f.tagName?(u.tag=f.tagName,u):(o&&(u.tag=o),u)}function uz(e,t,n,i,o){var v;const l=i.tag,u=l?t.directives.tagName(l.source,b=>o(l,"TAG_RESOLVE_FAILED",b)):null;if(n.type==="block-seq"){const{anchor:b,newlineAfterProp:w}=i,S=b&&l?b.offset>l.offset?b:l:b??l;S&&(!w||w.offsetb.tag===u&&b.collection===f);if(!d){const b=t.schema.knownTags[u];if(b&&b.collection===f)t.schema.tags.push(Object.assign({},b,{default:!1})),d=b;else return b!=null&&b.collection?o(l,"BAD_COLLECTION_TYPE",`${b.tag} used for ${f} collection, but expects ${b.collection}`,!0):o(l,"TAG_RESOLVE_FAILED",`Unresolved tag: ${u}`,!0),Th(e,t,n,o,u)}const p=Th(e,t,n,o,u,d),m=((v=d.resolve)==null?void 0:v.call(d,p,b=>o(l,"TAG_RESOLVE_FAILED",b),t.options))??p,g=Ht(m)?m:new Ie(m);return g.range=p.range,g.tag=u,d!=null&&d.format&&(g.format=d.format),g}function cz(e,t,n){const i=t.offset,o=fz(t,e.options.strict,n);if(!o)return{value:"",type:null,comment:"",range:[i,i,i]};const l=o.mode===">"?Ie.BLOCK_FOLDED:Ie.BLOCK_LITERAL,u=t.source?dz(t.source):[];let f=u.length;for(let S=u.length-1;S>=0;--S){const C=u[S][1];if(C===""||C==="\r")f=S;else break}if(f===0){const S=o.chomp==="+"&&u.length>0?` +`.repeat(Math.max(1,u.length-1)):"";let C=i+o.length;return t.source&&(C+=t.source.length),{value:S,type:l,comment:o.comment,range:[i,C,C]}}let d=t.indent+o.indent,p=t.offset+o.length,m=0;for(let S=0;Sd&&(d=C.length);else{C.length=f;--S)u[S][0].length>d&&(f=S+1);let g="",v="",b=!1;for(let S=0;Sd||A[0]===" "?(v===" "?v=` +`:!b&&v===` +`&&(v=` + +`),g+=v+C.slice(d)+A,v=` +`,b=!0):A===""?v===` +`?g+=` +`:v=` +`:(g+=v+A,v=" ",b=!1)}switch(o.chomp){case"-":break;case"+":for(let S=f;Sn(i+v,b,w);switch(o){case"scalar":f=Ie.PLAIN,d=hz(l,p);break;case"single-quoted-scalar":f=Ie.QUOTE_SINGLE,d=mz(l,p);break;case"double-quoted-scalar":f=Ie.QUOTE_DOUBLE,d=gz(l,p);break;default:return n(e,"UNEXPECTED_TOKEN",`Expected a flow scalar value, but found: ${o}`),{value:"",type:null,comment:"",range:[i,i+l.length,i+l.length]}}const m=i+l.length,g=tu(u,m,t,n);return{value:d,type:f,comment:g.comment,range:[i,m,g.offset]}}function hz(e,t){let n="";switch(e[0]){case" ":n="a tab character";break;case",":n="flow indicator character ,";break;case"%":n="directive indicator character %";break;case"|":case">":{n=`block scalar indicator ${e[0]}`;break}case"@":case"`":{n=`reserved character ${e[0]}`;break}}return n&&t(0,"BAD_SCALAR_START",`Plain value cannot start with ${n}`),mC(e)}function mz(e,t){return(e[e.length-1]!=="'"||e.length===1)&&t(e.length,"MISSING_CHAR","Missing closing 'quote"),mC(e.slice(1,-1)).replace(/''/g,"'")}function mC(e){let t,n;try{t=new RegExp(`(.*?)(?l?e.slice(l,i+1):o)}else n+=o}return(e[e.length-1]!=='"'||e.length===1)&&t(e.length,"MISSING_CHAR",'Missing closing "quote'),n}function yz(e,t){let n="",i=e[t+1];for(;(i===" "||i===" "||i===` +`||i==="\r")&&!(i==="\r"&&e[t+2]!==` +`);)i===` +`&&(n+=` +`),t+=1,i=e[t+1];return n||(n=" "),{fold:n,offset:t}}const bz={0:"\0",a:"\x07",b:"\b",e:"\x1B",f:"\f",n:` +`,r:"\r",t:" ",v:"\v",N:"…",_:" ",L:"\u2028",P:"\u2029"," ":" ",'"':'"',"/":"/","\\":"\\"," ":" "};function vz(e,t,n,i){const o=e.substr(t,n),u=o.length===n&&/^[0-9a-fA-F]+$/.test(o)?parseInt(o,16):NaN;if(isNaN(u)){const f=e.substr(t-2,n+2);return i(t-2,"BAD_DQ_ESCAPE",`Invalid escape sequence ${f}`),f}return String.fromCodePoint(u)}function gC(e,t,n,i){const{value:o,type:l,comment:u,range:f}=t.type==="block-scalar"?cz(e,t,i):pz(t,e.options.strict,i),d=n?e.directives.tagName(n.source,g=>i(n,"TAG_RESOLVE_FAILED",g)):null;let p;e.options.stringKeys&&e.atKey?p=e.schema[Vr]:d?p=Sz(e.schema,o,d,n,i):t.type==="scalar"?p=wz(e,o,t,i):p=e.schema[Vr];let m;try{const g=p.resolve(o,v=>i(n??t,"TAG_RESOLVE_FAILED",v),e.options);m=Tt(g)?g:new Ie(g)}catch(g){const v=g instanceof Error?g.message:String(g);i(n??t,"TAG_RESOLVE_FAILED",v),m=new Ie(o)}return m.range=f,m.source=o,l&&(m.type=l),d&&(m.tag=d),p.format&&(m.format=p.format),u&&(m.comment=u),m}function Sz(e,t,n,i,o){var f;if(n==="!")return e[Vr];const l=[];for(const d of e.tags)if(!d.collection&&d.tag===n)if(d.default&&d.test)l.push(d);else return d;for(const d of l)if((f=d.test)!=null&&f.test(t))return d;const u=e.knownTags[n];return u&&!u.collection?(e.tags.push(Object.assign({},u,{default:!1,test:void 0})),u):(o(i,"TAG_RESOLVE_FAILED",`Unresolved tag: ${n}`,n!=="tag:yaml.org,2002:str"),e[Vr])}function wz({atKey:e,directives:t,schema:n},i,o,l){const u=n.tags.find(f=>{var d;return(f.default===!0||e&&f.default==="key")&&((d=f.test)==null?void 0:d.test(i))})||n[Vr];if(n.compat){const f=n.compat.find(d=>{var p;return d.default&&((p=d.test)==null?void 0:p.test(i))})??n[Vr];if(u.tag!==f.tag){const d=t.tagString(u.tag),p=t.tagString(f.tag),m=`Value may be parsed as either ${d} or ${p}`;l(o,"TAG_RESOLVE_FAILED",m,!0)}}return u}function xz(e,t,n){if(t){n===null&&(n=t.length);for(let i=n-1;i>=0;--i){let o=t[i];switch(o.type){case"space":case"comment":case"newline":e-=o.source.length;continue}for(o=t[++i];(o==null?void 0:o.type)==="space";)e+=o.source.length,o=t[++i];break}}return e}const Cz={composeNode:yC,composeEmptyNode:Og};function yC(e,t,n,i){const o=e.atKey,{spaceBefore:l,comment:u,anchor:f,tag:d}=n;let p,m=!0;switch(t.type){case"alias":p=kz(e,t,i),(f||d)&&i(t,"ALIAS_PROPS","An alias node must not specify any properties");break;case"scalar":case"single-quoted-scalar":case"double-quoted-scalar":case"block-scalar":p=gC(e,t,d,i),f&&(p.anchor=f.source.substring(1));break;case"block-map":case"block-seq":case"flow-collection":p=uz(Cz,e,t,n,i),f&&(p.anchor=f.source.substring(1));break;default:{const g=t.type==="error"?t.message:`Unsupported token (type: ${t.type})`;i(t,"UNEXPECTED_TOKEN",g),p=Og(e,t.offset,void 0,null,n,i),m=!1}}return f&&p.anchor===""&&i(f,"BAD_ALIAS","Anchor cannot be an empty string"),o&&e.options.stringKeys&&(!Tt(p)||typeof p.value!="string"||p.tag&&p.tag!=="tag:yaml.org,2002:str")&&i(d??t,"NON_STRING_KEY","With stringKeys, all keys must be strings"),l&&(p.spaceBefore=!0),u&&(t.type==="scalar"&&t.source===""?p.comment=u:p.commentBefore=u),e.options.keepSourceTokens&&m&&(p.srcToken=t),p}function Og(e,t,n,i,{spaceBefore:o,comment:l,anchor:u,tag:f,end:d},p){const m={type:"scalar",offset:xz(t,n,i),indent:-1,source:""},g=gC(e,m,f,p);return u&&(g.anchor=u.source.substring(1),g.anchor===""&&p(u,"BAD_ALIAS","Anchor cannot be an empty string")),o&&(g.spaceBefore=!0),l&&(g.comment=l,g.range[2]=d),g}function kz({options:e},{offset:t,source:n,end:i},o){const l=new vg(n.substring(1));l.source===""&&o(t,"BAD_ALIAS","Alias cannot be an empty string"),l.source.endsWith(":")&&o(t+n.length-1,"BAD_ALIAS","Alias ending in : is ambiguous",!0);const u=t+n.length,f=tu(i,u,e.strict,o);return l.range=[t,u,f.offset],f.comment&&(l.comment=f.comment),l}function Ez(e,t,{offset:n,start:i,value:o,end:l},u){const f=Object.assign({_directives:t},e),d=new eu(void 0,f),p={atKey:!1,atRoot:!0,directives:d.directives,options:d.options,schema:d.schema},m=qo(i,{indicator:"doc-start",next:o??(l==null?void 0:l[0]),offset:n,onError:u,parentIndent:0,startOnNewline:!0});m.found&&(d.directives.docStart=!0,o&&(o.type==="block-map"||o.type==="block-seq")&&!m.hasNewline&&u(m.end,"MISSING_CHAR","Block collection cannot start on same line with directives-end marker")),d.contents=o?yC(p,o,m,u):Og(p,m.end,i,null,m,u);const g=d.contents.range[2],v=tu(l,g,!1,u);return v.comment&&(d.comment=v.comment),d.range=[n,g,v.offset],d}function al(e){if(typeof e=="number")return[e,e+1];if(Array.isArray(e))return e.length===2?e:[e[0],e[1]];const{offset:t,source:n}=e;return[t,t+(typeof n=="string"?n.length:1)]}function l1(e){var o;let t="",n=!1,i=!1;for(let l=0;l{const u=al(n);l?this.warnings.push(new az(u,i,o)):this.errors.push(new hl(u,i,o))},this.directives=new Cn({version:t.version||"1.2"}),this.options=t}decorate(t,n){const{comment:i,afterEmptyLine:o}=l1(this.prelude);if(i){const l=t.contents;if(n)t.comment=t.comment?`${t.comment} +${i}`:i;else if(o||t.directives.docStart||!l)t.commentBefore=i;else if(Ut(l)&&!l.flow&&l.items.length>0){let u=l.items[0];qt(u)&&(u=u.key);const f=u.commentBefore;u.commentBefore=f?`${i} +${f}`:i}else{const u=l.commentBefore;l.commentBefore=u?`${i} +${u}`:i}}n?(Array.prototype.push.apply(t.errors,this.errors),Array.prototype.push.apply(t.warnings,this.warnings)):(t.errors=this.errors,t.warnings=this.warnings),this.prelude=[],this.errors=[],this.warnings=[]}streamInfo(){return{comment:l1(this.prelude).comment,directives:this.directives,errors:this.errors,warnings:this.warnings}}*compose(t,n=!1,i=-1){for(const o of t)yield*this.next(o);yield*this.end(n,i)}*next(t){switch(t.type){case"directive":this.directives.add(t.source,(n,i,o)=>{const l=al(t);l[0]+=n,this.onError(l,"BAD_DIRECTIVE",i,o)}),this.prelude.push(t.source),this.atDirectives=!0;break;case"document":{const n=Ez(this.options,this.directives,t,this.onError);this.atDirectives&&!n.directives.docStart&&this.onError(t,"MISSING_CHAR","Missing directives-end/doc-start indicator line"),this.decorate(n,!1),this.doc&&(yield this.doc),this.doc=n,this.atDirectives=!1;break}case"byte-order-mark":case"space":break;case"comment":case"newline":this.prelude.push(t.source);break;case"error":{const n=t.source?`${t.message}: ${JSON.stringify(t.source)}`:t.message,i=new hl(al(t),"UNEXPECTED_TOKEN",n);this.atDirectives||!this.doc?this.errors.push(i):this.doc.errors.push(i);break}case"doc-end":{if(!this.doc){const i="Unexpected doc-end without preceding document";this.errors.push(new hl(al(t),"UNEXPECTED_TOKEN",i));break}this.doc.directives.docEnd=!0;const n=tu(t.end,t.offset+t.source.length,this.doc.options.strict,this.onError);if(this.decorate(this.doc,!0),n.comment){const i=this.doc.comment;this.doc.comment=i?`${i} +${n.comment}`:n.comment}this.doc.range[2]=n.offset;break}default:this.errors.push(new hl(al(t),"UNEXPECTED_TOKEN",`Unsupported token ${t.type}`))}}*end(t=!1,n=-1){if(this.doc)this.decorate(this.doc,!0),yield this.doc,this.doc=null;else if(t){const i=Object.assign({_directives:this.directives},this.options),o=new eu(void 0,i);this.atDirectives&&this.onError(n,"MISSING_CHAR","Missing directives-end indicator line"),o.range=[0,n,n],this.decorate(o,!1),yield o}}}const bC="\uFEFF",vC="",SC="",vm="";function Az(e){switch(e){case bC:return"byte-order-mark";case vC:return"doc-mode";case SC:return"flow-error-end";case vm:return"scalar";case"---":return"doc-start";case"...":return"doc-end";case"":case` +`:case`\r +`:return"newline";case"-":return"seq-item-ind";case"?":return"explicit-key-ind";case":":return"map-value-ind";case"{":return"flow-map-start";case"}":return"flow-map-end";case"[":return"flow-seq-start";case"]":return"flow-seq-end";case",":return"comma"}switch(e[0]){case" ":case" ":return"space";case"#":return"comment";case"%":return"directive-line";case"*":return"alias";case"&":return"anchor";case"!":return"tag";case"'":return"single-quoted-scalar";case'"':return"double-quoted-scalar";case"|":case">":return"block-scalar-header"}return null}function Sr(e){switch(e){case void 0:case" ":case` +`:case"\r":case" ":return!0;default:return!1}}const u1=new Set("0123456789ABCDEFabcdef"),_z=new Set("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-#;/?:@&=+$_.!~*'()"),xc=new Set(",[]{}"),Rz=new Set(` ,[]{} +\r `),Ah=e=>!e||Rz.has(e);class Oz{constructor(){this.atEnd=!1,this.blockScalarIndent=-1,this.blockScalarKeep=!1,this.buffer="",this.flowKey=!1,this.flowLevel=0,this.indentNext=0,this.indentValue=0,this.lineEndPos=null,this.next=null,this.pos=0}*lex(t,n=!1){if(t){if(typeof t!="string")throw TypeError("source is not a string");this.buffer=this.buffer?this.buffer+t:t,this.lineEndPos=null}this.atEnd=!n;let i=this.next??"stream";for(;i&&(n||this.hasChars(1));)i=yield*this.parseNext(i)}atLineEnd(){let t=this.pos,n=this.buffer[t];for(;n===" "||n===" ";)n=this.buffer[++t];return!n||n==="#"||n===` +`?!0:n==="\r"?this.buffer[t+1]===` +`:!1}charAt(t){return this.buffer[this.pos+t]}continueScalar(t){let n=this.buffer[t];if(this.indentNext>0){let i=0;for(;n===" ";)n=this.buffer[++i+t];if(n==="\r"){const o=this.buffer[i+t+1];if(o===` +`||!o&&!this.atEnd)return t+i+1}return n===` +`||i>=this.indentNext||!n&&!this.atEnd?t+i:-1}if(n==="-"||n==="."){const i=this.buffer.substr(t,3);if((i==="---"||i==="...")&&Sr(this.buffer[t+3]))return-1}return t}getLine(){let t=this.lineEndPos;return(typeof t!="number"||t!==-1&&tthis.indentValue&&!Sr(this.charAt(1))&&(this.indentNext=this.indentValue),yield*this.parseBlockStart()}*parseBlockStart(){const[t,n]=this.peek(2);if(!n&&!this.atEnd)return this.setNext("block-start");if((t==="-"||t==="?"||t===":")&&Sr(n)){const i=(yield*this.pushCount(1))+(yield*this.pushSpaces(!0));return this.indentNext=this.indentValue+1,this.indentValue+=i,yield*this.parseBlockStart()}return"doc"}*parseDocument(){yield*this.pushSpaces(!0);const t=this.getLine();if(t===null)return this.setNext("doc");let n=yield*this.pushIndicators();switch(t[n]){case"#":yield*this.pushCount(t.length-n);case void 0:return yield*this.pushNewline(),yield*this.parseLineStart();case"{":case"[":return yield*this.pushCount(1),this.flowKey=!1,this.flowLevel=1,"flow";case"}":case"]":return yield*this.pushCount(1),"doc";case"*":return yield*this.pushUntil(Ah),"doc";case'"':case"'":return yield*this.parseQuotedScalar();case"|":case">":return n+=yield*this.parseBlockScalarHeader(),n+=yield*this.pushSpaces(!0),yield*this.pushCount(t.length-n),yield*this.pushNewline(),yield*this.parseBlockScalar();default:return yield*this.parsePlainScalar()}}*parseFlowCollection(){let t,n,i=-1;do t=yield*this.pushNewline(),t>0?(n=yield*this.pushSpaces(!1),this.indentValue=i=n):n=0,n+=yield*this.pushSpaces(!0);while(t+n>0);const o=this.getLine();if(o===null)return this.setNext("flow");if((i!==-1&&i"0"&&n<="9")this.blockScalarIndent=Number(n)-1;else if(n!=="-")break}return yield*this.pushUntil(n=>Sr(n)||n==="#")}*parseBlockScalar(){let t=this.pos-1,n=0,i;e:for(let l=this.pos;i=this.buffer[l];++l)switch(i){case" ":n+=1;break;case` +`:t=l,n=0;break;case"\r":{const u=this.buffer[l+1];if(!u&&!this.atEnd)return this.setNext("block-scalar");if(u===` +`)break}default:break e}if(!i&&!this.atEnd)return this.setNext("block-scalar");if(n>=this.indentNext){this.blockScalarIndent===-1?this.indentNext=n:this.indentNext=this.blockScalarIndent+(this.indentNext===0?1:this.indentNext);do{const l=this.continueScalar(t+1);if(l===-1)break;t=this.buffer.indexOf(` +`,l)}while(t!==-1);if(t===-1){if(!this.atEnd)return this.setNext("block-scalar");t=this.buffer.length}}let o=t+1;for(i=this.buffer[o];i===" ";)i=this.buffer[++o];if(i===" "){for(;i===" "||i===" "||i==="\r"||i===` +`;)i=this.buffer[++o];t=o-1}else if(!this.blockScalarKeep)do{let l=t-1,u=this.buffer[l];u==="\r"&&(u=this.buffer[--l]);const f=l;for(;u===" ";)u=this.buffer[--l];if(u===` +`&&l>=this.pos&&l+1+n>f)t=l;else break}while(!0);return yield vm,yield*this.pushToIndex(t+1,!0),yield*this.parseLineStart()}*parsePlainScalar(){const t=this.flowLevel>0;let n=this.pos-1,i=this.pos-1,o;for(;o=this.buffer[++i];)if(o===":"){const l=this.buffer[i+1];if(Sr(l)||t&&xc.has(l))break;n=i}else if(Sr(o)){let l=this.buffer[i+1];if(o==="\r"&&(l===` +`?(i+=1,o=` +`,l=this.buffer[i+1]):n=i),l==="#"||t&&xc.has(l))break;if(o===` +`){const u=this.continueScalar(i+1);if(u===-1)break;i=Math.max(i,u-2)}}else{if(t&&xc.has(o))break;n=i}return!o&&!this.atEnd?this.setNext("plain-scalar"):(yield vm,yield*this.pushToIndex(n+1,!0),t?"flow":"doc")}*pushCount(t){return t>0?(yield this.buffer.substr(this.pos,t),this.pos+=t,t):0}*pushToIndex(t,n){const i=this.buffer.slice(this.pos,t);return i?(yield i,this.pos+=i.length,i.length):(n&&(yield""),0)}*pushIndicators(){switch(this.charAt(0)){case"!":return(yield*this.pushTag())+(yield*this.pushSpaces(!0))+(yield*this.pushIndicators());case"&":return(yield*this.pushUntil(Ah))+(yield*this.pushSpaces(!0))+(yield*this.pushIndicators());case"-":case"?":case":":{const t=this.flowLevel>0,n=this.charAt(1);if(Sr(n)||t&&xc.has(n))return t?this.flowKey&&(this.flowKey=!1):this.indentNext=this.indentValue+1,(yield*this.pushCount(1))+(yield*this.pushSpaces(!0))+(yield*this.pushIndicators())}}return 0}*pushTag(){if(this.charAt(1)==="<"){let t=this.pos+2,n=this.buffer[t];for(;!Sr(n)&&n!==">";)n=this.buffer[++t];return yield*this.pushToIndex(n===">"?t+1:t,!1)}else{let t=this.pos+1,n=this.buffer[t];for(;n;)if(_z.has(n))n=this.buffer[++t];else if(n==="%"&&u1.has(this.buffer[t+1])&&u1.has(this.buffer[t+2]))n=this.buffer[t+=3];else break;return yield*this.pushToIndex(t,!1)}}*pushNewline(){const t=this.buffer[this.pos];return t===` +`?yield*this.pushCount(1):t==="\r"&&this.charAt(1)===` +`?yield*this.pushCount(2):0}*pushSpaces(t){let n=this.pos-1,i;do i=this.buffer[++n];while(i===" "||t&&i===" ");const o=n-this.pos;return o>0&&(yield this.buffer.substr(this.pos,o),this.pos=n),o}*pushUntil(t){let n=this.pos,i=this.buffer[n];for(;!t(i);)i=this.buffer[++n];return yield*this.pushToIndex(n,!1)}}class Mz{constructor(){this.lineStarts=[],this.addNewLine=t=>this.lineStarts.push(t),this.linePos=t=>{let n=0,i=this.lineStarts.length;for(;n>1;this.lineStarts[l]=0;)switch(e[t].type){case"doc-start":case"explicit-key-ind":case"map-value-ind":case"seq-item-ind":case"newline":break e}for(;((n=e[++t])==null?void 0:n.type)==="space";);return e.splice(t,e.length)}function f1(e){if(e.start.type==="flow-seq-start")for(const t of e.items)t.sep&&!t.value&&!Ca(t.start,"explicit-key-ind")&&!Ca(t.sep,"map-value-ind")&&(t.key&&(t.value=t.key),delete t.key,wC(t.value)?t.value.end?Array.prototype.push.apply(t.value.end,t.sep):t.value.end=t.sep:Array.prototype.push.apply(t.start,t.sep),delete t.sep)}class Nz{constructor(t){this.atNewLine=!0,this.atScalar=!1,this.indent=0,this.offset=0,this.onKeyLine=!1,this.stack=[],this.source="",this.type="",this.lexer=new Oz,this.onNewLine=t}*parse(t,n=!1){this.onNewLine&&this.offset===0&&this.onNewLine(0);for(const i of this.lexer.lex(t,n))yield*this.next(i);n||(yield*this.end())}*next(t){if(this.source=t,this.atScalar){this.atScalar=!1,yield*this.step(),this.offset+=t.length;return}const n=Az(t);if(n)if(n==="scalar")this.atNewLine=!1,this.atScalar=!0,this.type="scalar";else{switch(this.type=n,yield*this.step(),n){case"newline":this.atNewLine=!0,this.indent=0,this.onNewLine&&this.onNewLine(this.offset+t.length);break;case"space":this.atNewLine&&t[0]===" "&&(this.indent+=t.length);break;case"explicit-key-ind":case"map-value-ind":case"seq-item-ind":this.atNewLine&&(this.indent+=t.length);break;case"doc-mode":case"flow-error-end":return;default:this.atNewLine=!1}this.offset+=t.length}else{const i=`Not a YAML token: ${t}`;yield*this.pop({type:"error",offset:this.offset,message:i,source:t}),this.offset+=t.length}}*end(){for(;this.stack.length>0;)yield*this.pop()}get sourceToken(){return{type:this.type,offset:this.offset,indent:this.indent,source:this.source}}*step(){const t=this.peek(1);if(this.type==="doc-end"&&(!t||t.type!=="doc-end")){for(;this.stack.length>0;)yield*this.pop();this.stack.push({type:"doc-end",offset:this.offset,source:this.source});return}if(!t)return yield*this.stream();switch(t.type){case"document":return yield*this.document(t);case"alias":case"scalar":case"single-quoted-scalar":case"double-quoted-scalar":return yield*this.scalar(t);case"block-scalar":return yield*this.blockScalar(t);case"block-map":return yield*this.blockMap(t);case"block-seq":return yield*this.blockSequence(t);case"flow-collection":return yield*this.flowCollection(t);case"doc-end":return yield*this.documentEnd(t)}yield*this.pop()}peek(t){return this.stack[this.stack.length-t]}*pop(t){const n=t??this.stack.pop();if(!n)yield{type:"error",offset:this.offset,source:"",message:"Tried to pop an empty stack"};else if(this.stack.length===0)yield n;else{const i=this.peek(1);switch(n.type==="block-scalar"?n.indent="indent"in i?i.indent:0:n.type==="flow-collection"&&i.type==="document"&&(n.indent=0),n.type==="flow-collection"&&f1(n),i.type){case"document":i.value=n;break;case"block-scalar":i.props.push(n);break;case"block-map":{const o=i.items[i.items.length-1];if(o.value){i.items.push({start:[],key:n,sep:[]}),this.onKeyLine=!0;return}else if(o.sep)o.value=n;else{Object.assign(o,{key:n,sep:[]}),this.onKeyLine=!o.explicitKey;return}break}case"block-seq":{const o=i.items[i.items.length-1];o.value?i.items.push({start:[],value:n}):o.value=n;break}case"flow-collection":{const o=i.items[i.items.length-1];!o||o.value?i.items.push({start:[],key:n,sep:[]}):o.sep?o.value=n:Object.assign(o,{key:n,sep:[]});return}default:yield*this.pop(),yield*this.pop(n)}if((i.type==="document"||i.type==="block-map"||i.type==="block-seq")&&(n.type==="block-map"||n.type==="block-seq")){const o=n.items[n.items.length-1];o&&!o.sep&&!o.value&&o.start.length>0&&c1(o.start)===-1&&(n.indent===0||o.start.every(l=>l.type!=="comment"||l.indent=t.indent){const o=!this.onKeyLine&&this.indent===t.indent,l=o&&(n.sep||n.explicitKey)&&this.type!=="seq-item-ind";let u=[];if(l&&n.sep&&!n.value){const f=[];for(let d=0;dt.indent&&(f.length=0);break;default:f.length=0}}f.length>=2&&(u=n.sep.splice(f[1]))}switch(this.type){case"anchor":case"tag":l||n.value?(u.push(this.sourceToken),t.items.push({start:u}),this.onKeyLine=!0):n.sep?n.sep.push(this.sourceToken):n.start.push(this.sourceToken);return;case"explicit-key-ind":!n.sep&&!n.explicitKey?(n.start.push(this.sourceToken),n.explicitKey=!0):l||n.value?(u.push(this.sourceToken),t.items.push({start:u,explicitKey:!0})):this.stack.push({type:"block-map",offset:this.offset,indent:this.indent,items:[{start:[this.sourceToken],explicitKey:!0}]}),this.onKeyLine=!0;return;case"map-value-ind":if(n.explicitKey)if(n.sep)if(n.value)t.items.push({start:[],key:null,sep:[this.sourceToken]});else if(Ca(n.sep,"map-value-ind"))this.stack.push({type:"block-map",offset:this.offset,indent:this.indent,items:[{start:u,key:null,sep:[this.sourceToken]}]});else if(wC(n.key)&&!Ca(n.sep,"newline")){const f=ko(n.start),d=n.key,p=n.sep;p.push(this.sourceToken),delete n.key,delete n.sep,this.stack.push({type:"block-map",offset:this.offset,indent:this.indent,items:[{start:f,key:d,sep:p}]})}else u.length>0?n.sep=n.sep.concat(u,this.sourceToken):n.sep.push(this.sourceToken);else if(Ca(n.start,"newline"))Object.assign(n,{key:null,sep:[this.sourceToken]});else{const f=ko(n.start);this.stack.push({type:"block-map",offset:this.offset,indent:this.indent,items:[{start:f,key:null,sep:[this.sourceToken]}]})}else n.sep?n.value||l?t.items.push({start:u,key:null,sep:[this.sourceToken]}):Ca(n.sep,"map-value-ind")?this.stack.push({type:"block-map",offset:this.offset,indent:this.indent,items:[{start:[],key:null,sep:[this.sourceToken]}]}):n.sep.push(this.sourceToken):Object.assign(n,{key:null,sep:[this.sourceToken]});this.onKeyLine=!0;return;case"alias":case"scalar":case"single-quoted-scalar":case"double-quoted-scalar":{const f=this.flowScalar(this.type);l||n.value?(t.items.push({start:u,key:f,sep:[]}),this.onKeyLine=!0):n.sep?this.stack.push(f):(Object.assign(n,{key:f,sep:[]}),this.onKeyLine=!0);return}default:{const f=this.startBlockValue(t);if(f){o&&f.type!=="block-seq"&&t.items.push({start:u}),this.stack.push(f);return}}}}yield*this.pop(),yield*this.step()}*blockSequence(t){var i;const n=t.items[t.items.length-1];switch(this.type){case"newline":if(n.value){const o="end"in n.value?n.value.end:void 0,l=Array.isArray(o)?o[o.length-1]:void 0;(l==null?void 0:l.type)==="comment"?o==null||o.push(this.sourceToken):t.items.push({start:[this.sourceToken]})}else n.start.push(this.sourceToken);return;case"space":case"comment":if(n.value)t.items.push({start:[this.sourceToken]});else{if(this.atIndentedComment(n.start,t.indent)){const o=t.items[t.items.length-2],l=(i=o==null?void 0:o.value)==null?void 0:i.end;if(Array.isArray(l)){Array.prototype.push.apply(l,n.start),l.push(this.sourceToken),t.items.pop();return}}n.start.push(this.sourceToken)}return;case"anchor":case"tag":if(n.value||this.indent<=t.indent)break;n.start.push(this.sourceToken);return;case"seq-item-ind":if(this.indent!==t.indent)break;n.value||Ca(n.start,"seq-item-ind")?t.items.push({start:[this.sourceToken]}):n.start.push(this.sourceToken);return}if(this.indent>t.indent){const o=this.startBlockValue(t);if(o){this.stack.push(o);return}}yield*this.pop(),yield*this.step()}*flowCollection(t){const n=t.items[t.items.length-1];if(this.type==="flow-error-end"){let i;do yield*this.pop(),i=this.peek(1);while(i&&i.type==="flow-collection")}else if(t.end.length===0){switch(this.type){case"comma":case"explicit-key-ind":!n||n.sep?t.items.push({start:[this.sourceToken]}):n.start.push(this.sourceToken);return;case"map-value-ind":!n||n.value?t.items.push({start:[],key:null,sep:[this.sourceToken]}):n.sep?n.sep.push(this.sourceToken):Object.assign(n,{key:null,sep:[this.sourceToken]});return;case"space":case"comment":case"newline":case"anchor":case"tag":!n||n.value?t.items.push({start:[this.sourceToken]}):n.sep?n.sep.push(this.sourceToken):n.start.push(this.sourceToken);return;case"alias":case"scalar":case"single-quoted-scalar":case"double-quoted-scalar":{const o=this.flowScalar(this.type);!n||n.value?t.items.push({start:[],key:o,sep:[]}):n.sep?this.stack.push(o):Object.assign(n,{key:o,sep:[]});return}case"flow-map-end":case"flow-seq-end":t.end.push(this.sourceToken);return}const i=this.startBlockValue(t);i?this.stack.push(i):(yield*this.pop(),yield*this.step())}else{const i=this.peek(2);if(i.type==="block-map"&&(this.type==="map-value-ind"&&i.indent===t.indent||this.type==="newline"&&!i.items[i.items.length-1].sep))yield*this.pop(),yield*this.step();else if(this.type==="map-value-ind"&&i.type!=="flow-collection"){const o=Cc(i),l=ko(o);f1(t);const u=t.end.splice(1,t.end.length);u.push(this.sourceToken);const f={type:"block-map",offset:t.offset,indent:t.indent,items:[{start:l,key:t,sep:u}]};this.onKeyLine=!0,this.stack[this.stack.length-1]=f}else yield*this.lineEnd(t)}}flowScalar(t){if(this.onNewLine){let n=this.source.indexOf(` +`)+1;for(;n!==0;)this.onNewLine(this.offset+n),n=this.source.indexOf(` +`,n)+1}return{type:t,offset:this.offset,indent:this.indent,source:this.source}}startBlockValue(t){switch(this.type){case"alias":case"scalar":case"single-quoted-scalar":case"double-quoted-scalar":return this.flowScalar(this.type);case"block-scalar-header":return{type:"block-scalar",offset:this.offset,indent:this.indent,props:[this.sourceToken],source:""};case"flow-map-start":case"flow-seq-start":return{type:"flow-collection",offset:this.offset,indent:this.indent,start:this.sourceToken,items:[],end:[]};case"seq-item-ind":return{type:"block-seq",offset:this.offset,indent:this.indent,items:[{start:[this.sourceToken]}]};case"explicit-key-ind":{this.onKeyLine=!0;const n=Cc(t),i=ko(n);return i.push(this.sourceToken),{type:"block-map",offset:this.offset,indent:this.indent,items:[{start:i,explicitKey:!0}]}}case"map-value-ind":{this.onKeyLine=!0;const n=Cc(t),i=ko(n);return{type:"block-map",offset:this.offset,indent:this.indent,items:[{start:i,key:null,sep:[this.sourceToken]}]}}}return null}atIndentedComment(t,n){return this.type!=="comment"||this.indent<=n?!1:t.every(i=>i.type==="newline"||i.type==="space")}*documentEnd(t){this.type!=="doc-mode"&&(t.end?t.end.push(this.sourceToken):t.end=[this.sourceToken],this.type==="newline"&&(yield*this.pop()))}*lineEnd(t){switch(this.type){case"comma":case"doc-start":case"doc-end":case"flow-seq-end":case"flow-map-end":case"map-value-ind":yield*this.pop(),yield*this.step();break;case"newline":this.onKeyLine=!1;case"space":case"comment":default:t.end?t.end.push(this.sourceToken):t.end=[this.sourceToken],this.type==="newline"&&(yield*this.pop())}}}function Dz(e){const t=e.prettyErrors!==!1;return{lineCounter:e.lineCounter||t&&new Mz||null,prettyErrors:t}}function xC(e,t={}){const{lineCounter:n,prettyErrors:i}=Dz(t),o=new Nz(n==null?void 0:n.addNewLine),l=new Tz(t);let u=null;for(const f of l.compose(o.parse(e),!0,e.length))if(!u)u=f;else if(u.options.logLevel!=="silent"){u.errors.push(new hl(f.range.slice(0,2),"MULTIPLE_DOCS","Source contains multiple documents; please use YAML.parseAllDocuments()"));break}return i&&n&&(u.errors.forEach(o1(e,n)),u.warnings.forEach(o1(e,n))),u}function Lz(e,t){const n={};return(e[e.length-1]===""?[...e,""]:e).join((n.padRight?" ":"")+","+(n.padLeft===!1?"":" ")).trim()}const zz=/^[$_\p{ID_Start}][$_\u{200C}\u{200D}\p{ID_Continue}]*$/u,Iz=/^[$_\p{ID_Start}][-$_\u{200C}\u{200D}\p{ID_Continue}]*$/u,Bz={};function d1(e,t){return(Bz.jsx?Iz:zz).test(e)}const jz=/[ \t\n\f\r]/g;function $z(e){return typeof e=="object"?e.type==="text"?p1(e.value):!1:p1(e)}function p1(e){return e.replace(jz,"")===""}class nu{constructor(t,n,i){this.normal=n,this.property=t,i&&(this.space=i)}}nu.prototype.normal={};nu.prototype.property={};nu.prototype.space=void 0;function CC(e,t){const n={},i={};for(const o of e)Object.assign(n,o.property),Object.assign(i,o.normal);return new nu(n,i,t)}function Sm(e){return e.toLowerCase()}class Bn{constructor(t,n){this.attribute=n,this.property=t}}Bn.prototype.attribute="";Bn.prototype.booleanish=!1;Bn.prototype.boolean=!1;Bn.prototype.commaOrSpaceSeparated=!1;Bn.prototype.commaSeparated=!1;Bn.prototype.defined=!1;Bn.prototype.mustUseProperty=!1;Bn.prototype.number=!1;Bn.prototype.overloadedBoolean=!1;Bn.prototype.property="";Bn.prototype.spaceSeparated=!1;Bn.prototype.space=void 0;let Pz=0;const Be=Na(),Xt=Na(),kC=Na(),pe=Na(),kt=Na(),jo=Na(),Vn=Na();function Na(){return 2**++Pz}const wm=Object.freeze(Object.defineProperty({__proto__:null,boolean:Be,booleanish:Xt,commaOrSpaceSeparated:Vn,commaSeparated:jo,number:pe,overloadedBoolean:kC,spaceSeparated:kt},Symbol.toStringTag,{value:"Module"})),_h=Object.keys(wm);class Mg extends Bn{constructor(t,n,i,o){let l=-1;if(super(t,n),h1(this,"space",o),typeof i=="number")for(;++l<_h.length;){const u=_h[l];h1(this,_h[l],(i&wm[u])===wm[u])}}}Mg.prototype.defined=!0;function h1(e,t,n){n&&(e[t]=n)}function es(e){const t={},n={};for(const[i,o]of Object.entries(e.properties)){const l=new Mg(i,e.transform(e.attributes||{},i),o,e.space);e.mustUseProperty&&e.mustUseProperty.includes(i)&&(l.mustUseProperty=!0),t[i]=l,n[Sm(i)]=i,n[Sm(l.attribute)]=i}return new nu(t,n,e.space)}const EC=es({properties:{ariaActiveDescendant:null,ariaAtomic:Xt,ariaAutoComplete:null,ariaBusy:Xt,ariaChecked:Xt,ariaColCount:pe,ariaColIndex:pe,ariaColSpan:pe,ariaControls:kt,ariaCurrent:null,ariaDescribedBy:kt,ariaDetails:null,ariaDisabled:Xt,ariaDropEffect:kt,ariaErrorMessage:null,ariaExpanded:Xt,ariaFlowTo:kt,ariaGrabbed:Xt,ariaHasPopup:null,ariaHidden:Xt,ariaInvalid:null,ariaKeyShortcuts:null,ariaLabel:null,ariaLabelledBy:kt,ariaLevel:pe,ariaLive:null,ariaModal:Xt,ariaMultiLine:Xt,ariaMultiSelectable:Xt,ariaOrientation:null,ariaOwns:kt,ariaPlaceholder:null,ariaPosInSet:pe,ariaPressed:Xt,ariaReadOnly:Xt,ariaRelevant:null,ariaRequired:Xt,ariaRoleDescription:kt,ariaRowCount:pe,ariaRowIndex:pe,ariaRowSpan:pe,ariaSelected:Xt,ariaSetSize:pe,ariaSort:null,ariaValueMax:pe,ariaValueMin:pe,ariaValueNow:pe,ariaValueText:null,role:null},transform(e,t){return t==="role"?t:"aria-"+t.slice(4).toLowerCase()}});function TC(e,t){return t in e?e[t]:t}function AC(e,t){return TC(e,t.toLowerCase())}const Uz=es({attributes:{acceptcharset:"accept-charset",classname:"class",htmlfor:"for",httpequiv:"http-equiv"},mustUseProperty:["checked","multiple","muted","selected"],properties:{abbr:null,accept:jo,acceptCharset:kt,accessKey:kt,action:null,allow:null,allowFullScreen:Be,allowPaymentRequest:Be,allowUserMedia:Be,alt:null,as:null,async:Be,autoCapitalize:null,autoComplete:kt,autoFocus:Be,autoPlay:Be,blocking:kt,capture:null,charSet:null,checked:Be,cite:null,className:kt,cols:pe,colSpan:null,content:null,contentEditable:Xt,controls:Be,controlsList:kt,coords:pe|jo,crossOrigin:null,data:null,dateTime:null,decoding:null,default:Be,defer:Be,dir:null,dirName:null,disabled:Be,download:kC,draggable:Xt,encType:null,enterKeyHint:null,fetchPriority:null,form:null,formAction:null,formEncType:null,formMethod:null,formNoValidate:Be,formTarget:null,headers:kt,height:pe,hidden:Be,high:pe,href:null,hrefLang:null,htmlFor:kt,httpEquiv:kt,id:null,imageSizes:null,imageSrcSet:null,inert:Be,inputMode:null,integrity:null,is:null,isMap:Be,itemId:null,itemProp:kt,itemRef:kt,itemScope:Be,itemType:kt,kind:null,label:null,lang:null,language:null,list:null,loading:null,loop:Be,low:pe,manifest:null,max:null,maxLength:pe,media:null,method:null,min:null,minLength:pe,multiple:Be,muted:Be,name:null,nonce:null,noModule:Be,noValidate:Be,onAbort:null,onAfterPrint:null,onAuxClick:null,onBeforeMatch:null,onBeforePrint:null,onBeforeToggle:null,onBeforeUnload:null,onBlur:null,onCancel:null,onCanPlay:null,onCanPlayThrough:null,onChange:null,onClick:null,onClose:null,onContextLost:null,onContextMenu:null,onContextRestored:null,onCopy:null,onCueChange:null,onCut:null,onDblClick:null,onDrag:null,onDragEnd:null,onDragEnter:null,onDragExit:null,onDragLeave:null,onDragOver:null,onDragStart:null,onDrop:null,onDurationChange:null,onEmptied:null,onEnded:null,onError:null,onFocus:null,onFormData:null,onHashChange:null,onInput:null,onInvalid:null,onKeyDown:null,onKeyPress:null,onKeyUp:null,onLanguageChange:null,onLoad:null,onLoadedData:null,onLoadedMetadata:null,onLoadEnd:null,onLoadStart:null,onMessage:null,onMessageError:null,onMouseDown:null,onMouseEnter:null,onMouseLeave:null,onMouseMove:null,onMouseOut:null,onMouseOver:null,onMouseUp:null,onOffline:null,onOnline:null,onPageHide:null,onPageShow:null,onPaste:null,onPause:null,onPlay:null,onPlaying:null,onPopState:null,onProgress:null,onRateChange:null,onRejectionHandled:null,onReset:null,onResize:null,onScroll:null,onScrollEnd:null,onSecurityPolicyViolation:null,onSeeked:null,onSeeking:null,onSelect:null,onSlotChange:null,onStalled:null,onStorage:null,onSubmit:null,onSuspend:null,onTimeUpdate:null,onToggle:null,onUnhandledRejection:null,onUnload:null,onVolumeChange:null,onWaiting:null,onWheel:null,open:Be,optimum:pe,pattern:null,ping:kt,placeholder:null,playsInline:Be,popover:null,popoverTarget:null,popoverTargetAction:null,poster:null,preload:null,readOnly:Be,referrerPolicy:null,rel:kt,required:Be,reversed:Be,rows:pe,rowSpan:pe,sandbox:kt,scope:null,scoped:Be,seamless:Be,selected:Be,shadowRootClonable:Be,shadowRootDelegatesFocus:Be,shadowRootMode:null,shape:null,size:pe,sizes:null,slot:null,span:pe,spellCheck:Xt,src:null,srcDoc:null,srcLang:null,srcSet:null,start:pe,step:null,style:null,tabIndex:pe,target:null,title:null,translate:null,type:null,typeMustMatch:Be,useMap:null,value:Xt,width:pe,wrap:null,writingSuggestions:null,align:null,aLink:null,archive:kt,axis:null,background:null,bgColor:null,border:pe,borderColor:null,bottomMargin:pe,cellPadding:null,cellSpacing:null,char:null,charOff:null,classId:null,clear:null,code:null,codeBase:null,codeType:null,color:null,compact:Be,declare:Be,event:null,face:null,frame:null,frameBorder:null,hSpace:pe,leftMargin:pe,link:null,longDesc:null,lowSrc:null,marginHeight:pe,marginWidth:pe,noResize:Be,noHref:Be,noShade:Be,noWrap:Be,object:null,profile:null,prompt:null,rev:null,rightMargin:pe,rules:null,scheme:null,scrolling:Xt,standby:null,summary:null,text:null,topMargin:pe,valueType:null,version:null,vAlign:null,vLink:null,vSpace:pe,allowTransparency:null,autoCorrect:null,autoSave:null,disablePictureInPicture:Be,disableRemotePlayback:Be,prefix:null,property:null,results:pe,security:null,unselectable:null},space:"html",transform:AC}),Hz=es({attributes:{accentHeight:"accent-height",alignmentBaseline:"alignment-baseline",arabicForm:"arabic-form",baselineShift:"baseline-shift",capHeight:"cap-height",className:"class",clipPath:"clip-path",clipRule:"clip-rule",colorInterpolation:"color-interpolation",colorInterpolationFilters:"color-interpolation-filters",colorProfile:"color-profile",colorRendering:"color-rendering",crossOrigin:"crossorigin",dataType:"datatype",dominantBaseline:"dominant-baseline",enableBackground:"enable-background",fillOpacity:"fill-opacity",fillRule:"fill-rule",floodColor:"flood-color",floodOpacity:"flood-opacity",fontFamily:"font-family",fontSize:"font-size",fontSizeAdjust:"font-size-adjust",fontStretch:"font-stretch",fontStyle:"font-style",fontVariant:"font-variant",fontWeight:"font-weight",glyphName:"glyph-name",glyphOrientationHorizontal:"glyph-orientation-horizontal",glyphOrientationVertical:"glyph-orientation-vertical",hrefLang:"hreflang",horizAdvX:"horiz-adv-x",horizOriginX:"horiz-origin-x",horizOriginY:"horiz-origin-y",imageRendering:"image-rendering",letterSpacing:"letter-spacing",lightingColor:"lighting-color",markerEnd:"marker-end",markerMid:"marker-mid",markerStart:"marker-start",navDown:"nav-down",navDownLeft:"nav-down-left",navDownRight:"nav-down-right",navLeft:"nav-left",navNext:"nav-next",navPrev:"nav-prev",navRight:"nav-right",navUp:"nav-up",navUpLeft:"nav-up-left",navUpRight:"nav-up-right",onAbort:"onabort",onActivate:"onactivate",onAfterPrint:"onafterprint",onBeforePrint:"onbeforeprint",onBegin:"onbegin",onCancel:"oncancel",onCanPlay:"oncanplay",onCanPlayThrough:"oncanplaythrough",onChange:"onchange",onClick:"onclick",onClose:"onclose",onCopy:"oncopy",onCueChange:"oncuechange",onCut:"oncut",onDblClick:"ondblclick",onDrag:"ondrag",onDragEnd:"ondragend",onDragEnter:"ondragenter",onDragExit:"ondragexit",onDragLeave:"ondragleave",onDragOver:"ondragover",onDragStart:"ondragstart",onDrop:"ondrop",onDurationChange:"ondurationchange",onEmptied:"onemptied",onEnd:"onend",onEnded:"onended",onError:"onerror",onFocus:"onfocus",onFocusIn:"onfocusin",onFocusOut:"onfocusout",onHashChange:"onhashchange",onInput:"oninput",onInvalid:"oninvalid",onKeyDown:"onkeydown",onKeyPress:"onkeypress",onKeyUp:"onkeyup",onLoad:"onload",onLoadedData:"onloadeddata",onLoadedMetadata:"onloadedmetadata",onLoadStart:"onloadstart",onMessage:"onmessage",onMouseDown:"onmousedown",onMouseEnter:"onmouseenter",onMouseLeave:"onmouseleave",onMouseMove:"onmousemove",onMouseOut:"onmouseout",onMouseOver:"onmouseover",onMouseUp:"onmouseup",onMouseWheel:"onmousewheel",onOffline:"onoffline",onOnline:"ononline",onPageHide:"onpagehide",onPageShow:"onpageshow",onPaste:"onpaste",onPause:"onpause",onPlay:"onplay",onPlaying:"onplaying",onPopState:"onpopstate",onProgress:"onprogress",onRateChange:"onratechange",onRepeat:"onrepeat",onReset:"onreset",onResize:"onresize",onScroll:"onscroll",onSeeked:"onseeked",onSeeking:"onseeking",onSelect:"onselect",onShow:"onshow",onStalled:"onstalled",onStorage:"onstorage",onSubmit:"onsubmit",onSuspend:"onsuspend",onTimeUpdate:"ontimeupdate",onToggle:"ontoggle",onUnload:"onunload",onVolumeChange:"onvolumechange",onWaiting:"onwaiting",onZoom:"onzoom",overlinePosition:"overline-position",overlineThickness:"overline-thickness",paintOrder:"paint-order",panose1:"panose-1",pointerEvents:"pointer-events",referrerPolicy:"referrerpolicy",renderingIntent:"rendering-intent",shapeRendering:"shape-rendering",stopColor:"stop-color",stopOpacity:"stop-opacity",strikethroughPosition:"strikethrough-position",strikethroughThickness:"strikethrough-thickness",strokeDashArray:"stroke-dasharray",strokeDashOffset:"stroke-dashoffset",strokeLineCap:"stroke-linecap",strokeLineJoin:"stroke-linejoin",strokeMiterLimit:"stroke-miterlimit",strokeOpacity:"stroke-opacity",strokeWidth:"stroke-width",tabIndex:"tabindex",textAnchor:"text-anchor",textDecoration:"text-decoration",textRendering:"text-rendering",transformOrigin:"transform-origin",typeOf:"typeof",underlinePosition:"underline-position",underlineThickness:"underline-thickness",unicodeBidi:"unicode-bidi",unicodeRange:"unicode-range",unitsPerEm:"units-per-em",vAlphabetic:"v-alphabetic",vHanging:"v-hanging",vIdeographic:"v-ideographic",vMathematical:"v-mathematical",vectorEffect:"vector-effect",vertAdvY:"vert-adv-y",vertOriginX:"vert-origin-x",vertOriginY:"vert-origin-y",wordSpacing:"word-spacing",writingMode:"writing-mode",xHeight:"x-height",playbackOrder:"playbackorder",timelineBegin:"timelinebegin"},properties:{about:Vn,accentHeight:pe,accumulate:null,additive:null,alignmentBaseline:null,alphabetic:pe,amplitude:pe,arabicForm:null,ascent:pe,attributeName:null,attributeType:null,azimuth:pe,bandwidth:null,baselineShift:null,baseFrequency:null,baseProfile:null,bbox:null,begin:null,bias:pe,by:null,calcMode:null,capHeight:pe,className:kt,clip:null,clipPath:null,clipPathUnits:null,clipRule:null,color:null,colorInterpolation:null,colorInterpolationFilters:null,colorProfile:null,colorRendering:null,content:null,contentScriptType:null,contentStyleType:null,crossOrigin:null,cursor:null,cx:null,cy:null,d:null,dataType:null,defaultAction:null,descent:pe,diffuseConstant:pe,direction:null,display:null,dur:null,divisor:pe,dominantBaseline:null,download:Be,dx:null,dy:null,edgeMode:null,editable:null,elevation:pe,enableBackground:null,end:null,event:null,exponent:pe,externalResourcesRequired:null,fill:null,fillOpacity:pe,fillRule:null,filter:null,filterRes:null,filterUnits:null,floodColor:null,floodOpacity:null,focusable:null,focusHighlight:null,fontFamily:null,fontSize:null,fontSizeAdjust:null,fontStretch:null,fontStyle:null,fontVariant:null,fontWeight:null,format:null,fr:null,from:null,fx:null,fy:null,g1:jo,g2:jo,glyphName:jo,glyphOrientationHorizontal:null,glyphOrientationVertical:null,glyphRef:null,gradientTransform:null,gradientUnits:null,handler:null,hanging:pe,hatchContentUnits:null,hatchUnits:null,height:null,href:null,hrefLang:null,horizAdvX:pe,horizOriginX:pe,horizOriginY:pe,id:null,ideographic:pe,imageRendering:null,initialVisibility:null,in:null,in2:null,intercept:pe,k:pe,k1:pe,k2:pe,k3:pe,k4:pe,kernelMatrix:Vn,kernelUnitLength:null,keyPoints:null,keySplines:null,keyTimes:null,kerning:null,lang:null,lengthAdjust:null,letterSpacing:null,lightingColor:null,limitingConeAngle:pe,local:null,markerEnd:null,markerMid:null,markerStart:null,markerHeight:null,markerUnits:null,markerWidth:null,mask:null,maskContentUnits:null,maskUnits:null,mathematical:null,max:null,media:null,mediaCharacterEncoding:null,mediaContentEncodings:null,mediaSize:pe,mediaTime:null,method:null,min:null,mode:null,name:null,navDown:null,navDownLeft:null,navDownRight:null,navLeft:null,navNext:null,navPrev:null,navRight:null,navUp:null,navUpLeft:null,navUpRight:null,numOctaves:null,observer:null,offset:null,onAbort:null,onActivate:null,onAfterPrint:null,onBeforePrint:null,onBegin:null,onCancel:null,onCanPlay:null,onCanPlayThrough:null,onChange:null,onClick:null,onClose:null,onCopy:null,onCueChange:null,onCut:null,onDblClick:null,onDrag:null,onDragEnd:null,onDragEnter:null,onDragExit:null,onDragLeave:null,onDragOver:null,onDragStart:null,onDrop:null,onDurationChange:null,onEmptied:null,onEnd:null,onEnded:null,onError:null,onFocus:null,onFocusIn:null,onFocusOut:null,onHashChange:null,onInput:null,onInvalid:null,onKeyDown:null,onKeyPress:null,onKeyUp:null,onLoad:null,onLoadedData:null,onLoadedMetadata:null,onLoadStart:null,onMessage:null,onMouseDown:null,onMouseEnter:null,onMouseLeave:null,onMouseMove:null,onMouseOut:null,onMouseOver:null,onMouseUp:null,onMouseWheel:null,onOffline:null,onOnline:null,onPageHide:null,onPageShow:null,onPaste:null,onPause:null,onPlay:null,onPlaying:null,onPopState:null,onProgress:null,onRateChange:null,onRepeat:null,onReset:null,onResize:null,onScroll:null,onSeeked:null,onSeeking:null,onSelect:null,onShow:null,onStalled:null,onStorage:null,onSubmit:null,onSuspend:null,onTimeUpdate:null,onToggle:null,onUnload:null,onVolumeChange:null,onWaiting:null,onZoom:null,opacity:null,operator:null,order:null,orient:null,orientation:null,origin:null,overflow:null,overlay:null,overlinePosition:pe,overlineThickness:pe,paintOrder:null,panose1:null,path:null,pathLength:pe,patternContentUnits:null,patternTransform:null,patternUnits:null,phase:null,ping:kt,pitch:null,playbackOrder:null,pointerEvents:null,points:null,pointsAtX:pe,pointsAtY:pe,pointsAtZ:pe,preserveAlpha:null,preserveAspectRatio:null,primitiveUnits:null,propagate:null,property:Vn,r:null,radius:null,referrerPolicy:null,refX:null,refY:null,rel:Vn,rev:Vn,renderingIntent:null,repeatCount:null,repeatDur:null,requiredExtensions:Vn,requiredFeatures:Vn,requiredFonts:Vn,requiredFormats:Vn,resource:null,restart:null,result:null,rotate:null,rx:null,ry:null,scale:null,seed:null,shapeRendering:null,side:null,slope:null,snapshotTime:null,specularConstant:pe,specularExponent:pe,spreadMethod:null,spacing:null,startOffset:null,stdDeviation:null,stemh:null,stemv:null,stitchTiles:null,stopColor:null,stopOpacity:null,strikethroughPosition:pe,strikethroughThickness:pe,string:null,stroke:null,strokeDashArray:Vn,strokeDashOffset:null,strokeLineCap:null,strokeLineJoin:null,strokeMiterLimit:pe,strokeOpacity:pe,strokeWidth:null,style:null,surfaceScale:pe,syncBehavior:null,syncBehaviorDefault:null,syncMaster:null,syncTolerance:null,syncToleranceDefault:null,systemLanguage:Vn,tabIndex:pe,tableValues:null,target:null,targetX:pe,targetY:pe,textAnchor:null,textDecoration:null,textRendering:null,textLength:null,timelineBegin:null,title:null,transformBehavior:null,type:null,typeOf:Vn,to:null,transform:null,transformOrigin:null,u1:null,u2:null,underlinePosition:pe,underlineThickness:pe,unicode:null,unicodeBidi:null,unicodeRange:null,unitsPerEm:pe,values:null,vAlphabetic:pe,vMathematical:pe,vectorEffect:null,vHanging:pe,vIdeographic:pe,version:null,vertAdvY:pe,vertOriginX:pe,vertOriginY:pe,viewBox:null,viewTarget:null,visibility:null,width:null,widths:null,wordSpacing:null,writingMode:null,x:null,x1:null,x2:null,xChannelSelector:null,xHeight:pe,y:null,y1:null,y2:null,yChannelSelector:null,z:null,zoomAndPan:null},space:"svg",transform:TC}),_C=es({properties:{xLinkActuate:null,xLinkArcRole:null,xLinkHref:null,xLinkRole:null,xLinkShow:null,xLinkTitle:null,xLinkType:null},space:"xlink",transform(e,t){return"xlink:"+t.slice(5).toLowerCase()}}),RC=es({attributes:{xmlnsxlink:"xmlns:xlink"},properties:{xmlnsXLink:null,xmlns:null},space:"xmlns",transform:AC}),OC=es({properties:{xmlBase:null,xmlLang:null,xmlSpace:null},space:"xml",transform(e,t){return"xml:"+t.slice(3).toLowerCase()}}),qz={classId:"classID",dataType:"datatype",itemId:"itemID",strokeDashArray:"strokeDasharray",strokeDashOffset:"strokeDashoffset",strokeLineCap:"strokeLinecap",strokeLineJoin:"strokeLinejoin",strokeMiterLimit:"strokeMiterlimit",typeOf:"typeof",xLinkActuate:"xlinkActuate",xLinkArcRole:"xlinkArcrole",xLinkHref:"xlinkHref",xLinkRole:"xlinkRole",xLinkShow:"xlinkShow",xLinkTitle:"xlinkTitle",xLinkType:"xlinkType",xmlnsXLink:"xmlnsXlink"},Fz=/[A-Z]/g,m1=/-[a-z]/g,Vz=/^data[-\w.:]+$/i;function Gz(e,t){const n=Sm(t);let i=t,o=Bn;if(n in e.normal)return e.property[e.normal[n]];if(n.length>4&&n.slice(0,4)==="data"&&Vz.test(t)){if(t.charAt(4)==="-"){const l=t.slice(5).replace(m1,Yz);i="data"+l.charAt(0).toUpperCase()+l.slice(1)}else{const l=t.slice(4);if(!m1.test(l)){let u=l.replace(Fz,Kz);u.charAt(0)!=="-"&&(u="-"+u),t="data"+u}}o=Mg}return new o(i,t)}function Kz(e){return"-"+e.toLowerCase()}function Yz(e){return e.charAt(1).toUpperCase()}const Xz=CC([EC,Uz,_C,RC,OC],"html"),Ng=CC([EC,Hz,_C,RC,OC],"svg");function Wz(e){return e.join(" ").trim()}var Eo={},Rh,g1;function Qz(){if(g1)return Rh;g1=1;var e=/\/\*[^*]*\*+([^/*][^*]*\*+)*\//g,t=/\n/g,n=/^\s*/,i=/^(\*?[-#/*\\\w]+(\[[0-9a-z_-]+\])?)\s*/,o=/^:\s*/,l=/^((?:'(?:\\'|.)*?'|"(?:\\"|.)*?"|\([^)]*?\)|[^};])+)/,u=/^[;\s]*/,f=/^\s+|\s+$/g,d=` +`,p="/",m="*",g="",v="comment",b="declaration";Rh=function(S,C){if(typeof S!="string")throw new TypeError("First argument must be a string");if(!S)return[];C=C||{};var A=1,O=1;function _(G){var Y=G.match(t);Y&&(A+=Y.length);var V=G.lastIndexOf(d);O=~V?G.length-V:O+G.length}function M(){var G={line:A,column:O};return function(Y){return Y.position=new R(G),z(),Y}}function R(G){this.start=G,this.end={line:A,column:O},this.source=C.source}R.prototype.content=S;function D(G){var Y=new Error(C.source+":"+A+":"+O+": "+G);if(Y.reason=G,Y.filename=C.source,Y.line=A,Y.column=O,Y.source=S,!C.silent)throw Y}function P(G){var Y=G.exec(S);if(Y){var V=Y[0];return _(V),S=S.slice(V.length),Y}}function z(){P(n)}function $(G){var Y;for(G=G||[];Y=E();)Y!==!1&&G.push(Y);return G}function E(){var G=M();if(!(p!=S.charAt(0)||m!=S.charAt(1))){for(var Y=2;g!=S.charAt(Y)&&(m!=S.charAt(Y)||p!=S.charAt(Y+1));)++Y;if(Y+=2,g===S.charAt(Y-1))return D("End of comment missing");var V=S.slice(2,Y-2);return O+=2,_(V),S=S.slice(Y),O+=2,G({type:v,comment:V})}}function I(){var G=M(),Y=P(i);if(Y){if(E(),!P(o))return D("property missing ':'");var V=P(l),H=G({type:b,property:w(Y[0].replace(e,g)),value:V?w(V[0].replace(e,g)):g});return P(u),H}}function U(){var G=[];$(G);for(var Y;Y=I();)Y!==!1&&(G.push(Y),$(G));return G}return z(),U()};function w(S){return S?S.replace(f,g):g}return Rh}var y1;function Zz(){if(y1)return Eo;y1=1;var e=Eo&&Eo.__importDefault||function(i){return i&&i.__esModule?i:{default:i}};Object.defineProperty(Eo,"__esModule",{value:!0}),Eo.default=n;var t=e(Qz());function n(i,o){var l=null;if(!i||typeof i!="string")return l;var u=(0,t.default)(i),f=typeof o=="function";return u.forEach(function(d){if(d.type==="declaration"){var p=d.property,m=d.value;f?o(p,m,d):m&&(l=l||{},l[p]=m)}}),l}return Eo}var Jz=Zz();const b1=rf(Jz),e4=b1.default||b1,MC=NC("end"),Dg=NC("start");function NC(e){return t;function t(n){const i=n&&n.position&&n.position[e]||{};if(typeof i.line=="number"&&i.line>0&&typeof i.column=="number"&&i.column>0)return{line:i.line,column:i.column,offset:typeof i.offset=="number"&&i.offset>-1?i.offset:void 0}}}function t4(e){const t=Dg(e),n=MC(e);if(t&&n)return{start:t,end:n}}function Cl(e){return!e||typeof e!="object"?"":"position"in e||"type"in e?v1(e.position):"start"in e||"end"in e?v1(e):"line"in e||"column"in e?xm(e):""}function xm(e){return S1(e&&e.line)+":"+S1(e&&e.column)}function v1(e){return xm(e&&e.start)+"-"+xm(e&&e.end)}function S1(e){return e&&typeof e=="number"?e:1}class vn extends Error{constructor(t,n,i){super(),typeof n=="string"&&(i=n,n=void 0);let o="",l={},u=!1;if(n&&("line"in n&&"column"in n?l={place:n}:"start"in n&&"end"in n?l={place:n}:"type"in n?l={ancestors:[n],place:n.position}:l={...n}),typeof t=="string"?o=t:!l.cause&&t&&(u=!0,o=t.message,l.cause=t),!l.ruleId&&!l.source&&typeof i=="string"){const d=i.indexOf(":");d===-1?l.ruleId=i:(l.source=i.slice(0,d),l.ruleId=i.slice(d+1))}if(!l.place&&l.ancestors&&l.ancestors){const d=l.ancestors[l.ancestors.length-1];d&&(l.place=d.position)}const f=l.place&&"start"in l.place?l.place.start:l.place;this.ancestors=l.ancestors||void 0,this.cause=l.cause||void 0,this.column=f?f.column:void 0,this.fatal=void 0,this.file,this.message=o,this.line=f?f.line:void 0,this.name=Cl(l.place)||"1:1",this.place=l.place||void 0,this.reason=this.message,this.ruleId=l.ruleId||void 0,this.source=l.source||void 0,this.stack=u&&l.cause&&typeof l.cause.stack=="string"?l.cause.stack:"",this.actual,this.expected,this.note,this.url}}vn.prototype.file="";vn.prototype.name="";vn.prototype.reason="";vn.prototype.message="";vn.prototype.stack="";vn.prototype.column=void 0;vn.prototype.line=void 0;vn.prototype.ancestors=void 0;vn.prototype.cause=void 0;vn.prototype.fatal=void 0;vn.prototype.place=void 0;vn.prototype.ruleId=void 0;vn.prototype.source=void 0;const Lg={}.hasOwnProperty,n4=new Map,r4=/[A-Z]/g,i4=/-([a-z])/g,a4=new Set(["table","tbody","thead","tfoot","tr"]),o4=new Set(["td","th"]),DC="https://github.com/syntax-tree/hast-util-to-jsx-runtime";function s4(e,t){if(!t||t.Fragment===void 0)throw new TypeError("Expected `Fragment` in options");const n=t.filePath||void 0;let i;if(t.development){if(typeof t.jsxDEV!="function")throw new TypeError("Expected `jsxDEV` in options when `development: true`");i=m4(n,t.jsxDEV)}else{if(typeof t.jsx!="function")throw new TypeError("Expected `jsx` in production options");if(typeof t.jsxs!="function")throw new TypeError("Expected `jsxs` in production options");i=h4(n,t.jsx,t.jsxs)}const o={Fragment:t.Fragment,ancestors:[],components:t.components||{},create:i,elementAttributeNameCase:t.elementAttributeNameCase||"react",evaluater:t.createEvaluater?t.createEvaluater():void 0,filePath:n,ignoreInvalidStyle:t.ignoreInvalidStyle||!1,passKeys:t.passKeys!==!1,passNode:t.passNode||!1,schema:t.space==="svg"?Ng:Xz,stylePropertyNameCase:t.stylePropertyNameCase||"dom",tableCellAlignToStyle:t.tableCellAlignToStyle!==!1},l=LC(o,e,void 0);return l&&typeof l!="string"?l:o.create(e,o.Fragment,{children:l||void 0},void 0)}function LC(e,t,n){if(t.type==="element")return l4(e,t,n);if(t.type==="mdxFlowExpression"||t.type==="mdxTextExpression")return u4(e,t);if(t.type==="mdxJsxFlowElement"||t.type==="mdxJsxTextElement")return f4(e,t,n);if(t.type==="mdxjsEsm")return c4(e,t);if(t.type==="root")return d4(e,t,n);if(t.type==="text")return p4(e,t)}function l4(e,t,n){const i=e.schema;let o=i;t.tagName.toLowerCase()==="svg"&&i.space==="html"&&(o=Ng,e.schema=o),e.ancestors.push(t);const l=IC(e,t.tagName,!1),u=g4(e,t);let f=Ig(e,t);return a4.has(t.tagName)&&(f=f.filter(function(d){return typeof d=="string"?!$z(d):!0})),zC(e,u,l,t),zg(u,f),e.ancestors.pop(),e.schema=i,e.create(t,l,u,n)}function u4(e,t){if(t.data&&t.data.estree&&e.evaluater){const i=t.data.estree.body[0];return i.type,e.evaluater.evaluateExpression(i.expression)}Il(e,t.position)}function c4(e,t){if(t.data&&t.data.estree&&e.evaluater)return e.evaluater.evaluateProgram(t.data.estree);Il(e,t.position)}function f4(e,t,n){const i=e.schema;let o=i;t.name==="svg"&&i.space==="html"&&(o=Ng,e.schema=o),e.ancestors.push(t);const l=t.name===null?e.Fragment:IC(e,t.name,!0),u=y4(e,t),f=Ig(e,t);return zC(e,u,l,t),zg(u,f),e.ancestors.pop(),e.schema=i,e.create(t,l,u,n)}function d4(e,t,n){const i={};return zg(i,Ig(e,t)),e.create(t,e.Fragment,i,n)}function p4(e,t){return t.value}function zC(e,t,n,i){typeof n!="string"&&n!==e.Fragment&&e.passNode&&(t.node=i)}function zg(e,t){if(t.length>0){const n=t.length>1?t:t[0];n&&(e.children=n)}}function h4(e,t,n){return i;function i(o,l,u,f){const p=Array.isArray(u.children)?n:t;return f?p(l,u,f):p(l,u)}}function m4(e,t){return n;function n(i,o,l,u){const f=Array.isArray(l.children),d=Dg(i);return t(o,l,u,f,{columnNumber:d?d.column-1:void 0,fileName:e,lineNumber:d?d.line:void 0},void 0)}}function g4(e,t){const n={};let i,o;for(o in t.properties)if(o!=="children"&&Lg.call(t.properties,o)){const l=b4(e,o,t.properties[o]);if(l){const[u,f]=l;e.tableCellAlignToStyle&&u==="align"&&typeof f=="string"&&o4.has(t.tagName)?i=f:n[u]=f}}if(i){const l=n.style||(n.style={});l[e.stylePropertyNameCase==="css"?"text-align":"textAlign"]=i}return n}function y4(e,t){const n={};for(const i of t.attributes)if(i.type==="mdxJsxExpressionAttribute")if(i.data&&i.data.estree&&e.evaluater){const l=i.data.estree.body[0];l.type;const u=l.expression;u.type;const f=u.properties[0];f.type,Object.assign(n,e.evaluater.evaluateExpression(f.argument))}else Il(e,t.position);else{const o=i.name;let l;if(i.value&&typeof i.value=="object")if(i.value.data&&i.value.data.estree&&e.evaluater){const f=i.value.data.estree.body[0];f.type,l=e.evaluater.evaluateExpression(f.expression)}else Il(e,t.position);else l=i.value===null?!0:i.value;n[o]=l}return n}function Ig(e,t){const n=[];let i=-1;const o=e.passKeys?new Map:n4;for(;++io?0:o+t:t=t>o?o:t,n=n>0?n:0,i.length<1e4)u=Array.from(i),u.unshift(t,n),e.splice(...u);else for(n&&e.splice(t,n);l0?(Gr(e,e.length,0,t),e):t}const C1={}.hasOwnProperty;function A4(e){const t={};let n=-1;for(;++n13&&n<32||n>126&&n<160||n>55295&&n<57344||n>64975&&n<65008||(n&65535)===65535||(n&65535)===65534||n>1114111?"�":String.fromCodePoint(n)}function $o(e){return e.replace(/[\t\n\r ]+/g," ").replace(/^ | $/g,"").toLowerCase().toUpperCase()}const Hr=Wi(/[A-Za-z]/),Kn=Wi(/[\dA-Za-z]/),O4=Wi(/[#-'*+\--9=?A-Z^-~]/);function Cm(e){return e!==null&&(e<32||e===127)}const km=Wi(/\d/),M4=Wi(/[\dA-Fa-f]/),N4=Wi(/[!-/:-@[-`{-~]/);function Le(e){return e!==null&&e<-2}function Ln(e){return e!==null&&(e<0||e===32)}function ct(e){return e===-2||e===-1||e===32}const D4=Wi(new RegExp("\\p{P}|\\p{S}","u")),L4=Wi(/\s/);function Wi(e){return t;function t(n){return n!==null&&n>-1&&e.test(String.fromCharCode(n))}}function ts(e){const t=[];let n=-1,i=0,o=0;for(;++n55295&&l<57344){const f=e.charCodeAt(n+1);l<56320&&f>56319&&f<57344?(u=String.fromCharCode(l,f),o=1):u="�"}else u=String.fromCharCode(l);u&&(t.push(e.slice(i,n),encodeURIComponent(u)),i=n+o+1,u=""),o&&(n+=o,o=0)}return t.join("")+e.slice(i)}function Et(e,t,n,i){const o=i?i-1:Number.POSITIVE_INFINITY;let l=0;return u;function u(d){return ct(d)?(e.enter(n),f(d)):t(d)}function f(d){return ct(d)&&l++u))return;const P=t.events.length;let z=P,$,E;for(;z--;)if(t.events[z][0]==="exit"&&t.events[z][1].type==="chunkFlow"){if($){E=t.events[z][1].end;break}$=!0}for(A(i),D=P;D_;){const R=n[M];t.containerState=R[1],R[0].exit.call(t,e)}n.length=_}function O(){o.write([null]),l=void 0,o=void 0,t.containerState._closeFlow=void 0}}function $4(e,t,n){return Et(e,e.attempt(this.parser.constructs.document,t,n),"linePrefix",this.parser.constructs.disable.null.includes("codeIndented")?void 0:4)}function E1(e){if(e===null||Ln(e)||L4(e))return 1;if(D4(e))return 2}function jg(e,t,n){const i=[];let o=-1;for(;++o1&&e[n][1].end.offset-e[n][1].start.offset>1?2:1;const g={...e[i][1].end},v={...e[n][1].start};T1(g,-d),T1(v,d),u={type:d>1?"strongSequence":"emphasisSequence",start:g,end:{...e[i][1].end}},f={type:d>1?"strongSequence":"emphasisSequence",start:{...e[n][1].start},end:v},l={type:d>1?"strongText":"emphasisText",start:{...e[i][1].end},end:{...e[n][1].start}},o={type:d>1?"strong":"emphasis",start:{...u.start},end:{...f.end}},e[i][1].end={...u.start},e[n][1].start={...f.end},p=[],e[i][1].end.offset-e[i][1].start.offset&&(p=fr(p,[["enter",e[i][1],t],["exit",e[i][1],t]])),p=fr(p,[["enter",o,t],["enter",u,t],["exit",u,t],["enter",l,t]]),p=fr(p,jg(t.parser.constructs.insideSpan.null,e.slice(i+1,n),t)),p=fr(p,[["exit",l,t],["enter",f,t],["exit",f,t],["exit",o,t]]),e[n][1].end.offset-e[n][1].start.offset?(m=2,p=fr(p,[["enter",e[n][1],t],["exit",e[n][1],t]])):m=0,Gr(e,i-1,n-i+3,p),n=i+p.length-m-2;break}}for(n=-1;++n0&&ct(D)?Et(e,O,"linePrefix",l+1)(D):O(D)}function O(D){return D===null||Le(D)?e.check(A1,S,M)(D):(e.enter("codeFlowValue"),_(D))}function _(D){return D===null||Le(D)?(e.exit("codeFlowValue"),O(D)):(e.consume(D),_)}function M(D){return e.exit("codeFenced"),t(D)}function R(D,P,z){let $=0;return E;function E(V){return D.enter("lineEnding"),D.consume(V),D.exit("lineEnding"),I}function I(V){return D.enter("codeFencedFence"),ct(V)?Et(D,U,"linePrefix",i.parser.constructs.disable.null.includes("codeIndented")?void 0:4)(V):U(V)}function U(V){return V===f?(D.enter("codeFencedFenceSequence"),G(V)):z(V)}function G(V){return V===f?($++,D.consume(V),G):$>=u?(D.exit("codeFencedFenceSequence"),ct(V)?Et(D,Y,"whitespace")(V):Y(V)):z(V)}function Y(V){return V===null||Le(V)?(D.exit("codeFencedFence"),P(V)):z(V)}}}function Q4(e,t,n){const i=this;return o;function o(u){return u===null?n(u):(e.enter("lineEnding"),e.consume(u),e.exit("lineEnding"),l)}function l(u){return i.parser.lazy[i.now().line]?n(u):t(u)}}const Mh={name:"codeIndented",tokenize:J4},Z4={partial:!0,tokenize:eI};function J4(e,t,n){const i=this;return o;function o(p){return e.enter("codeIndented"),Et(e,l,"linePrefix",5)(p)}function l(p){const m=i.events[i.events.length-1];return m&&m[1].type==="linePrefix"&&m[2].sliceSerialize(m[1],!0).length>=4?u(p):n(p)}function u(p){return p===null?d(p):Le(p)?e.attempt(Z4,u,d)(p):(e.enter("codeFlowValue"),f(p))}function f(p){return p===null||Le(p)?(e.exit("codeFlowValue"),u(p)):(e.consume(p),f)}function d(p){return e.exit("codeIndented"),t(p)}}function eI(e,t,n){const i=this;return o;function o(u){return i.parser.lazy[i.now().line]?n(u):Le(u)?(e.enter("lineEnding"),e.consume(u),e.exit("lineEnding"),o):Et(e,l,"linePrefix",5)(u)}function l(u){const f=i.events[i.events.length-1];return f&&f[1].type==="linePrefix"&&f[2].sliceSerialize(f[1],!0).length>=4?t(u):Le(u)?o(u):n(u)}}const tI={name:"codeText",previous:rI,resolve:nI,tokenize:iI};function nI(e){let t=e.length-4,n=3,i,o;if((e[n][1].type==="lineEnding"||e[n][1].type==="space")&&(e[t][1].type==="lineEnding"||e[t][1].type==="space")){for(i=n;++i=this.left.length+this.right.length)throw new RangeError("Cannot access index `"+t+"` in a splice buffer of size `"+(this.left.length+this.right.length)+"`");return tthis.left.length?this.right.slice(this.right.length-i+this.left.length,this.right.length-t+this.left.length).reverse():this.left.slice(t).concat(this.right.slice(this.right.length-i+this.left.length).reverse())}splice(t,n,i){const o=n||0;this.setCursor(Math.trunc(t));const l=this.right.splice(this.right.length-o,Number.POSITIVE_INFINITY);return i&&ol(this.left,i),l.reverse()}pop(){return this.setCursor(Number.POSITIVE_INFINITY),this.left.pop()}push(t){this.setCursor(Number.POSITIVE_INFINITY),this.left.push(t)}pushMany(t){this.setCursor(Number.POSITIVE_INFINITY),ol(this.left,t)}unshift(t){this.setCursor(0),this.right.push(t)}unshiftMany(t){this.setCursor(0),ol(this.right,t.reverse())}setCursor(t){if(!(t===this.left.length||t>this.left.length&&this.right.length===0||t<0&&this.left.length===0))if(t=4?t(u):e.interrupt(i.parser.constructs.flow,n,t)(u)}}function qC(e,t,n,i,o,l,u,f,d){const p=d||Number.POSITIVE_INFINITY;let m=0;return g;function g(A){return A===60?(e.enter(i),e.enter(o),e.enter(l),e.consume(A),e.exit(l),v):A===null||A===32||A===41||Cm(A)?n(A):(e.enter(i),e.enter(u),e.enter(f),e.enter("chunkString",{contentType:"string"}),S(A))}function v(A){return A===62?(e.enter(l),e.consume(A),e.exit(l),e.exit(o),e.exit(i),t):(e.enter(f),e.enter("chunkString",{contentType:"string"}),b(A))}function b(A){return A===62?(e.exit("chunkString"),e.exit(f),v(A)):A===null||A===60||Le(A)?n(A):(e.consume(A),A===92?w:b)}function w(A){return A===60||A===62||A===92?(e.consume(A),b):b(A)}function S(A){return!m&&(A===null||A===41||Ln(A))?(e.exit("chunkString"),e.exit(f),e.exit(u),e.exit(i),t(A)):m999||b===null||b===91||b===93&&!d||b===94&&!f&&"_hiddenFootnoteSupport"in u.parser.constructs?n(b):b===93?(e.exit(l),e.enter(o),e.consume(b),e.exit(o),e.exit(i),t):Le(b)?(e.enter("lineEnding"),e.consume(b),e.exit("lineEnding"),m):(e.enter("chunkString",{contentType:"string"}),g(b))}function g(b){return b===null||b===91||b===93||Le(b)||f++>999?(e.exit("chunkString"),m(b)):(e.consume(b),d||(d=!ct(b)),b===92?v:g)}function v(b){return b===91||b===92||b===93?(e.consume(b),f++,g):g(b)}}function VC(e,t,n,i,o,l){let u;return f;function f(v){return v===34||v===39||v===40?(e.enter(i),e.enter(o),e.consume(v),e.exit(o),u=v===40?41:v,d):n(v)}function d(v){return v===u?(e.enter(o),e.consume(v),e.exit(o),e.exit(i),t):(e.enter(l),p(v))}function p(v){return v===u?(e.exit(l),d(u)):v===null?n(v):Le(v)?(e.enter("lineEnding"),e.consume(v),e.exit("lineEnding"),Et(e,p,"linePrefix")):(e.enter("chunkString",{contentType:"string"}),m(v))}function m(v){return v===u||v===null||Le(v)?(e.exit("chunkString"),p(v)):(e.consume(v),v===92?g:m)}function g(v){return v===u||v===92?(e.consume(v),m):m(v)}}function kl(e,t){let n;return i;function i(o){return Le(o)?(e.enter("lineEnding"),e.consume(o),e.exit("lineEnding"),n=!0,i):ct(o)?Et(e,i,n?"linePrefix":"lineSuffix")(o):t(o)}}const dI={name:"definition",tokenize:hI},pI={partial:!0,tokenize:mI};function hI(e,t,n){const i=this;let o;return l;function l(b){return e.enter("definition"),u(b)}function u(b){return FC.call(i,e,f,n,"definitionLabel","definitionLabelMarker","definitionLabelString")(b)}function f(b){return o=$o(i.sliceSerialize(i.events[i.events.length-1][1]).slice(1,-1)),b===58?(e.enter("definitionMarker"),e.consume(b),e.exit("definitionMarker"),d):n(b)}function d(b){return Ln(b)?kl(e,p)(b):p(b)}function p(b){return qC(e,m,n,"definitionDestination","definitionDestinationLiteral","definitionDestinationLiteralMarker","definitionDestinationRaw","definitionDestinationString")(b)}function m(b){return e.attempt(pI,g,g)(b)}function g(b){return ct(b)?Et(e,v,"whitespace")(b):v(b)}function v(b){return b===null||Le(b)?(e.exit("definition"),i.parser.defined.push(o),t(b)):n(b)}}function mI(e,t,n){return i;function i(f){return Ln(f)?kl(e,o)(f):n(f)}function o(f){return VC(e,l,n,"definitionTitle","definitionTitleMarker","definitionTitleString")(f)}function l(f){return ct(f)?Et(e,u,"whitespace")(f):u(f)}function u(f){return f===null||Le(f)?t(f):n(f)}}const gI={name:"hardBreakEscape",tokenize:yI};function yI(e,t,n){return i;function i(l){return e.enter("hardBreakEscape"),e.consume(l),o}function o(l){return Le(l)?(e.exit("hardBreakEscape"),t(l)):n(l)}}const bI={name:"headingAtx",resolve:vI,tokenize:SI};function vI(e,t){let n=e.length-2,i=3,o,l;return e[i][1].type==="whitespace"&&(i+=2),n-2>i&&e[n][1].type==="whitespace"&&(n-=2),e[n][1].type==="atxHeadingSequence"&&(i===n-1||n-4>i&&e[n-2][1].type==="whitespace")&&(n-=i+1===n?2:4),n>i&&(o={type:"atxHeadingText",start:e[i][1].start,end:e[n][1].end},l={type:"chunkText",start:e[i][1].start,end:e[n][1].end,contentType:"text"},Gr(e,i,n-i+1,[["enter",o,t],["enter",l,t],["exit",l,t],["exit",o,t]])),e}function SI(e,t,n){let i=0;return o;function o(m){return e.enter("atxHeading"),l(m)}function l(m){return e.enter("atxHeadingSequence"),u(m)}function u(m){return m===35&&i++<6?(e.consume(m),u):m===null||Ln(m)?(e.exit("atxHeadingSequence"),f(m)):n(m)}function f(m){return m===35?(e.enter("atxHeadingSequence"),d(m)):m===null||Le(m)?(e.exit("atxHeading"),t(m)):ct(m)?Et(e,f,"whitespace")(m):(e.enter("atxHeadingText"),p(m))}function d(m){return m===35?(e.consume(m),d):(e.exit("atxHeadingSequence"),f(m))}function p(m){return m===null||m===35||Ln(m)?(e.exit("atxHeadingText"),f(m)):(e.consume(m),p)}}const wI=["address","article","aside","base","basefont","blockquote","body","caption","center","col","colgroup","dd","details","dialog","dir","div","dl","dt","fieldset","figcaption","figure","footer","form","frame","frameset","h1","h2","h3","h4","h5","h6","head","header","hr","html","iframe","legend","li","link","main","menu","menuitem","nav","noframes","ol","optgroup","option","p","param","search","section","summary","table","tbody","td","tfoot","th","thead","title","tr","track","ul"],R1=["pre","script","style","textarea"],xI={concrete:!0,name:"htmlFlow",resolveTo:EI,tokenize:TI},CI={partial:!0,tokenize:_I},kI={partial:!0,tokenize:AI};function EI(e){let t=e.length;for(;t--&&!(e[t][0]==="enter"&&e[t][1].type==="htmlFlow"););return t>1&&e[t-2][1].type==="linePrefix"&&(e[t][1].start=e[t-2][1].start,e[t+1][1].start=e[t-2][1].start,e.splice(t-2,2)),e}function TI(e,t,n){const i=this;let o,l,u,f,d;return p;function p(L){return m(L)}function m(L){return e.enter("htmlFlow"),e.enter("htmlFlowData"),e.consume(L),g}function g(L){return L===33?(e.consume(L),v):L===47?(e.consume(L),l=!0,S):L===63?(e.consume(L),o=3,i.interrupt?t:k):Hr(L)?(e.consume(L),u=String.fromCharCode(L),C):n(L)}function v(L){return L===45?(e.consume(L),o=2,b):L===91?(e.consume(L),o=5,f=0,w):Hr(L)?(e.consume(L),o=4,i.interrupt?t:k):n(L)}function b(L){return L===45?(e.consume(L),i.interrupt?t:k):n(L)}function w(L){const re="CDATA[";return L===re.charCodeAt(f++)?(e.consume(L),f===re.length?i.interrupt?t:U:w):n(L)}function S(L){return Hr(L)?(e.consume(L),u=String.fromCharCode(L),C):n(L)}function C(L){if(L===null||L===47||L===62||Ln(L)){const re=L===47,fe=u.toLowerCase();return!re&&!l&&R1.includes(fe)?(o=1,i.interrupt?t(L):U(L)):wI.includes(u.toLowerCase())?(o=6,re?(e.consume(L),A):i.interrupt?t(L):U(L)):(o=7,i.interrupt&&!i.parser.lazy[i.now().line]?n(L):l?O(L):_(L))}return L===45||Kn(L)?(e.consume(L),u+=String.fromCharCode(L),C):n(L)}function A(L){return L===62?(e.consume(L),i.interrupt?t:U):n(L)}function O(L){return ct(L)?(e.consume(L),O):E(L)}function _(L){return L===47?(e.consume(L),E):L===58||L===95||Hr(L)?(e.consume(L),M):ct(L)?(e.consume(L),_):E(L)}function M(L){return L===45||L===46||L===58||L===95||Kn(L)?(e.consume(L),M):R(L)}function R(L){return L===61?(e.consume(L),D):ct(L)?(e.consume(L),R):_(L)}function D(L){return L===null||L===60||L===61||L===62||L===96?n(L):L===34||L===39?(e.consume(L),d=L,P):ct(L)?(e.consume(L),D):z(L)}function P(L){return L===d?(e.consume(L),d=null,$):L===null||Le(L)?n(L):(e.consume(L),P)}function z(L){return L===null||L===34||L===39||L===47||L===60||L===61||L===62||L===96||Ln(L)?R(L):(e.consume(L),z)}function $(L){return L===47||L===62||ct(L)?_(L):n(L)}function E(L){return L===62?(e.consume(L),I):n(L)}function I(L){return L===null||Le(L)?U(L):ct(L)?(e.consume(L),I):n(L)}function U(L){return L===45&&o===2?(e.consume(L),H):L===60&&o===1?(e.consume(L),Z):L===62&&o===4?(e.consume(L),X):L===63&&o===3?(e.consume(L),k):L===93&&o===5?(e.consume(L),J):Le(L)&&(o===6||o===7)?(e.exit("htmlFlowData"),e.check(CI,se,G)(L)):L===null||Le(L)?(e.exit("htmlFlowData"),G(L)):(e.consume(L),U)}function G(L){return e.check(kI,Y,se)(L)}function Y(L){return e.enter("lineEnding"),e.consume(L),e.exit("lineEnding"),V}function V(L){return L===null||Le(L)?G(L):(e.enter("htmlFlowData"),U(L))}function H(L){return L===45?(e.consume(L),k):U(L)}function Z(L){return L===47?(e.consume(L),u="",ae):U(L)}function ae(L){if(L===62){const re=u.toLowerCase();return R1.includes(re)?(e.consume(L),X):U(L)}return Hr(L)&&u.length<8?(e.consume(L),u+=String.fromCharCode(L),ae):U(L)}function J(L){return L===93?(e.consume(L),k):U(L)}function k(L){return L===62?(e.consume(L),X):L===45&&o===2?(e.consume(L),k):U(L)}function X(L){return L===null||Le(L)?(e.exit("htmlFlowData"),se(L)):(e.consume(L),X)}function se(L){return e.exit("htmlFlow"),t(L)}}function AI(e,t,n){const i=this;return o;function o(u){return Le(u)?(e.enter("lineEnding"),e.consume(u),e.exit("lineEnding"),l):n(u)}function l(u){return i.parser.lazy[i.now().line]?n(u):t(u)}}function _I(e,t,n){return i;function i(o){return e.enter("lineEnding"),e.consume(o),e.exit("lineEnding"),e.attempt(Hf,t,n)}}const RI={name:"htmlText",tokenize:OI};function OI(e,t,n){const i=this;let o,l,u;return f;function f(k){return e.enter("htmlText"),e.enter("htmlTextData"),e.consume(k),d}function d(k){return k===33?(e.consume(k),p):k===47?(e.consume(k),R):k===63?(e.consume(k),_):Hr(k)?(e.consume(k),z):n(k)}function p(k){return k===45?(e.consume(k),m):k===91?(e.consume(k),l=0,w):Hr(k)?(e.consume(k),O):n(k)}function m(k){return k===45?(e.consume(k),b):n(k)}function g(k){return k===null?n(k):k===45?(e.consume(k),v):Le(k)?(u=g,Z(k)):(e.consume(k),g)}function v(k){return k===45?(e.consume(k),b):g(k)}function b(k){return k===62?H(k):k===45?v(k):g(k)}function w(k){const X="CDATA[";return k===X.charCodeAt(l++)?(e.consume(k),l===X.length?S:w):n(k)}function S(k){return k===null?n(k):k===93?(e.consume(k),C):Le(k)?(u=S,Z(k)):(e.consume(k),S)}function C(k){return k===93?(e.consume(k),A):S(k)}function A(k){return k===62?H(k):k===93?(e.consume(k),A):S(k)}function O(k){return k===null||k===62?H(k):Le(k)?(u=O,Z(k)):(e.consume(k),O)}function _(k){return k===null?n(k):k===63?(e.consume(k),M):Le(k)?(u=_,Z(k)):(e.consume(k),_)}function M(k){return k===62?H(k):_(k)}function R(k){return Hr(k)?(e.consume(k),D):n(k)}function D(k){return k===45||Kn(k)?(e.consume(k),D):P(k)}function P(k){return Le(k)?(u=P,Z(k)):ct(k)?(e.consume(k),P):H(k)}function z(k){return k===45||Kn(k)?(e.consume(k),z):k===47||k===62||Ln(k)?$(k):n(k)}function $(k){return k===47?(e.consume(k),H):k===58||k===95||Hr(k)?(e.consume(k),E):Le(k)?(u=$,Z(k)):ct(k)?(e.consume(k),$):H(k)}function E(k){return k===45||k===46||k===58||k===95||Kn(k)?(e.consume(k),E):I(k)}function I(k){return k===61?(e.consume(k),U):Le(k)?(u=I,Z(k)):ct(k)?(e.consume(k),I):$(k)}function U(k){return k===null||k===60||k===61||k===62||k===96?n(k):k===34||k===39?(e.consume(k),o=k,G):Le(k)?(u=U,Z(k)):ct(k)?(e.consume(k),U):(e.consume(k),Y)}function G(k){return k===o?(e.consume(k),o=void 0,V):k===null?n(k):Le(k)?(u=G,Z(k)):(e.consume(k),G)}function Y(k){return k===null||k===34||k===39||k===60||k===61||k===96?n(k):k===47||k===62||Ln(k)?$(k):(e.consume(k),Y)}function V(k){return k===47||k===62||Ln(k)?$(k):n(k)}function H(k){return k===62?(e.consume(k),e.exit("htmlTextData"),e.exit("htmlText"),t):n(k)}function Z(k){return e.exit("htmlTextData"),e.enter("lineEnding"),e.consume(k),e.exit("lineEnding"),ae}function ae(k){return ct(k)?Et(e,J,"linePrefix",i.parser.constructs.disable.null.includes("codeIndented")?void 0:4)(k):J(k)}function J(k){return e.enter("htmlTextData"),u(k)}}const $g={name:"labelEnd",resolveAll:LI,resolveTo:zI,tokenize:II},MI={tokenize:BI},NI={tokenize:jI},DI={tokenize:$I};function LI(e){let t=-1;const n=[];for(;++t=3&&(p===null||Le(p))?(e.exit("thematicBreak"),t(p)):n(p)}function d(p){return p===o?(e.consume(p),i++,d):(e.exit("thematicBreakSequence"),ct(p)?Et(e,f,"whitespace")(p):f(p))}}const Dn={continuation:{tokenize:XI},exit:QI,name:"list",tokenize:YI},GI={partial:!0,tokenize:ZI},KI={partial:!0,tokenize:WI};function YI(e,t,n){const i=this,o=i.events[i.events.length-1];let l=o&&o[1].type==="linePrefix"?o[2].sliceSerialize(o[1],!0).length:0,u=0;return f;function f(b){const w=i.containerState.type||(b===42||b===43||b===45?"listUnordered":"listOrdered");if(w==="listUnordered"?!i.containerState.marker||b===i.containerState.marker:km(b)){if(i.containerState.type||(i.containerState.type=w,e.enter(w,{_container:!0})),w==="listUnordered")return e.enter("listItemPrefix"),b===42||b===45?e.check(Bc,n,p)(b):p(b);if(!i.interrupt||b===49)return e.enter("listItemPrefix"),e.enter("listItemValue"),d(b)}return n(b)}function d(b){return km(b)&&++u<10?(e.consume(b),d):(!i.interrupt||u<2)&&(i.containerState.marker?b===i.containerState.marker:b===41||b===46)?(e.exit("listItemValue"),p(b)):n(b)}function p(b){return e.enter("listItemMarker"),e.consume(b),e.exit("listItemMarker"),i.containerState.marker=i.containerState.marker||b,e.check(Hf,i.interrupt?n:m,e.attempt(GI,v,g))}function m(b){return i.containerState.initialBlankLine=!0,l++,v(b)}function g(b){return ct(b)?(e.enter("listItemPrefixWhitespace"),e.consume(b),e.exit("listItemPrefixWhitespace"),v):n(b)}function v(b){return i.containerState.size=l+i.sliceSerialize(e.exit("listItemPrefix"),!0).length,t(b)}}function XI(e,t,n){const i=this;return i.containerState._closeFlow=void 0,e.check(Hf,o,l);function o(f){return i.containerState.furtherBlankLines=i.containerState.furtherBlankLines||i.containerState.initialBlankLine,Et(e,t,"listItemIndent",i.containerState.size+1)(f)}function l(f){return i.containerState.furtherBlankLines||!ct(f)?(i.containerState.furtherBlankLines=void 0,i.containerState.initialBlankLine=void 0,u(f)):(i.containerState.furtherBlankLines=void 0,i.containerState.initialBlankLine=void 0,e.attempt(KI,t,u)(f))}function u(f){return i.containerState._closeFlow=!0,i.interrupt=void 0,Et(e,e.attempt(Dn,t,n),"linePrefix",i.parser.constructs.disable.null.includes("codeIndented")?void 0:4)(f)}}function WI(e,t,n){const i=this;return Et(e,o,"listItemIndent",i.containerState.size+1);function o(l){const u=i.events[i.events.length-1];return u&&u[1].type==="listItemIndent"&&u[2].sliceSerialize(u[1],!0).length===i.containerState.size?t(l):n(l)}}function QI(e){e.exit(this.containerState.type)}function ZI(e,t,n){const i=this;return Et(e,o,"listItemPrefixWhitespace",i.parser.constructs.disable.null.includes("codeIndented")?void 0:5);function o(l){const u=i.events[i.events.length-1];return!ct(l)&&u&&u[1].type==="listItemPrefixWhitespace"?t(l):n(l)}}const O1={name:"setextUnderline",resolveTo:JI,tokenize:e3};function JI(e,t){let n=e.length,i,o,l;for(;n--;)if(e[n][0]==="enter"){if(e[n][1].type==="content"){i=n;break}e[n][1].type==="paragraph"&&(o=n)}else e[n][1].type==="content"&&e.splice(n,1),!l&&e[n][1].type==="definition"&&(l=n);const u={type:"setextHeading",start:{...e[o][1].start},end:{...e[e.length-1][1].end}};return e[o][1].type="setextHeadingText",l?(e.splice(o,0,["enter",u,t]),e.splice(l+1,0,["exit",e[i][1],t]),e[i][1].end={...e[l][1].end}):e[i][1]=u,e.push(["exit",u,t]),e}function e3(e,t,n){const i=this;let o;return l;function l(p){let m=i.events.length,g;for(;m--;)if(i.events[m][1].type!=="lineEnding"&&i.events[m][1].type!=="linePrefix"&&i.events[m][1].type!=="content"){g=i.events[m][1].type==="paragraph";break}return!i.parser.lazy[i.now().line]&&(i.interrupt||g)?(e.enter("setextHeadingLine"),o=p,u(p)):n(p)}function u(p){return e.enter("setextHeadingLineSequence"),f(p)}function f(p){return p===o?(e.consume(p),f):(e.exit("setextHeadingLineSequence"),ct(p)?Et(e,d,"lineSuffix")(p):d(p))}function d(p){return p===null||Le(p)?(e.exit("setextHeadingLine"),t(p)):n(p)}}const t3={tokenize:n3};function n3(e){const t=this,n=e.attempt(Hf,i,e.attempt(this.parser.constructs.flowInitial,o,Et(e,e.attempt(this.parser.constructs.flow,o,e.attempt(sI,o)),"linePrefix")));return n;function i(l){if(l===null){e.consume(l);return}return e.enter("lineEndingBlank"),e.consume(l),e.exit("lineEndingBlank"),t.currentConstruct=void 0,n}function o(l){if(l===null){e.consume(l);return}return e.enter("lineEnding"),e.consume(l),e.exit("lineEnding"),t.currentConstruct=void 0,n}}const r3={resolveAll:KC()},i3=GC("string"),a3=GC("text");function GC(e){return{resolveAll:KC(e==="text"?o3:void 0),tokenize:t};function t(n){const i=this,o=this.parser.constructs[e],l=n.attempt(o,u,f);return u;function u(m){return p(m)?l(m):f(m)}function f(m){if(m===null){n.consume(m);return}return n.enter("data"),n.consume(m),d}function d(m){return p(m)?(n.exit("data"),l(m)):(n.consume(m),d)}function p(m){if(m===null)return!0;const g=o[m];let v=-1;if(g)for(;++v-1){const f=u[0];typeof f=="string"?u[0]=f.slice(i):u.shift()}l>0&&u.push(e[o].slice(0,l))}return u}function v3(e,t){let n=-1;const i=[];let o;for(;++n0){const Me=he.tokenStack[he.tokenStack.length-1];(Me[1]||N1).call(he,void 0,Me[0])}for(de.position={start:Fi(ee.length>0?ee[0][1].start:{line:1,column:1,offset:0}),end:Fi(ee.length>0?ee[ee.length-2][1].end:{line:1,column:1,offset:0})},Re=-1;++Re1?"-"+f:""),dataFootnoteRef:!0,ariaDescribedBy:["footnote-label"]},children:[{type:"text",value:String(u)}]};e.patch(t,d);const p={type:"element",tagName:"sup",properties:{},children:[d]};return e.patch(t,p),e.applyData(t,p)}function I3(e,t){const n={type:"element",tagName:"h"+t.depth,properties:{},children:e.all(t)};return e.patch(t,n),e.applyData(t,n)}function B3(e,t){if(e.options.allowDangerousHtml){const n={type:"raw",value:t.value};return e.patch(t,n),e.applyData(t,n)}}function WC(e,t){const n=t.referenceType;let i="]";if(n==="collapsed"?i+="[]":n==="full"&&(i+="["+(t.label||t.identifier)+"]"),t.type==="imageReference")return[{type:"text",value:"!["+t.alt+i}];const o=e.all(t),l=o[0];l&&l.type==="text"?l.value="["+l.value:o.unshift({type:"text",value:"["});const u=o[o.length-1];return u&&u.type==="text"?u.value+=i:o.push({type:"text",value:i}),o}function j3(e,t){const n=String(t.identifier).toUpperCase(),i=e.definitionById.get(n);if(!i)return WC(e,t);const o={src:ts(i.url||""),alt:t.alt};i.title!==null&&i.title!==void 0&&(o.title=i.title);const l={type:"element",tagName:"img",properties:o,children:[]};return e.patch(t,l),e.applyData(t,l)}function $3(e,t){const n={src:ts(t.url)};t.alt!==null&&t.alt!==void 0&&(n.alt=t.alt),t.title!==null&&t.title!==void 0&&(n.title=t.title);const i={type:"element",tagName:"img",properties:n,children:[]};return e.patch(t,i),e.applyData(t,i)}function P3(e,t){const n={type:"text",value:t.value.replace(/\r?\n|\r/g," ")};e.patch(t,n);const i={type:"element",tagName:"code",properties:{},children:[n]};return e.patch(t,i),e.applyData(t,i)}function U3(e,t){const n=String(t.identifier).toUpperCase(),i=e.definitionById.get(n);if(!i)return WC(e,t);const o={href:ts(i.url||"")};i.title!==null&&i.title!==void 0&&(o.title=i.title);const l={type:"element",tagName:"a",properties:o,children:e.all(t)};return e.patch(t,l),e.applyData(t,l)}function H3(e,t){const n={href:ts(t.url)};t.title!==null&&t.title!==void 0&&(n.title=t.title);const i={type:"element",tagName:"a",properties:n,children:e.all(t)};return e.patch(t,i),e.applyData(t,i)}function q3(e,t,n){const i=e.all(t),o=n?F3(n):QC(t),l={},u=[];if(typeof t.checked=="boolean"){const m=i[0];let g;m&&m.type==="element"&&m.tagName==="p"?g=m:(g={type:"element",tagName:"p",properties:{},children:[]},i.unshift(g)),g.children.length>0&&g.children.unshift({type:"text",value:" "}),g.children.unshift({type:"element",tagName:"input",properties:{type:"checkbox",checked:t.checked,disabled:!0},children:[]}),l.className=["task-list-item"]}let f=-1;for(;++f1}function V3(e,t){const n={},i=e.all(t);let o=-1;for(typeof t.start=="number"&&t.start!==1&&(n.start=t.start);++o0){const u={type:"element",tagName:"tbody",properties:{},children:e.wrap(n,!0)},f=Dg(t.children[1]),d=MC(t.children[t.children.length-1]);f&&d&&(u.position={start:f,end:d}),o.push(u)}const l={type:"element",tagName:"table",properties:{},children:e.wrap(o,!0)};return e.patch(t,l),e.applyData(t,l)}function W3(e,t,n){const i=n?n.children:void 0,l=(i?i.indexOf(t):1)===0?"th":"td",u=n&&n.type==="table"?n.align:void 0,f=u?u.length:t.children.length;let d=-1;const p=[];for(;++d0,!0),i[0]),o=i.index+i[0].length,i=n.exec(t);return l.push(z1(t.slice(o),o>0,!1)),l.join("")}function z1(e,t,n){let i=0,o=e.length;if(t){let l=e.codePointAt(i);for(;l===D1||l===L1;)i++,l=e.codePointAt(i)}if(n){let l=e.codePointAt(o-1);for(;l===D1||l===L1;)o--,l=e.codePointAt(o-1)}return o>i?e.slice(i,o):""}function J3(e,t){const n={type:"text",value:Z3(String(t.value))};return e.patch(t,n),e.applyData(t,n)}function eB(e,t){const n={type:"element",tagName:"hr",properties:{},children:[]};return e.patch(t,n),e.applyData(t,n)}const tB={blockquote:O3,break:M3,code:N3,delete:D3,emphasis:L3,footnoteReference:z3,heading:I3,html:B3,imageReference:j3,image:$3,inlineCode:P3,linkReference:U3,link:H3,listItem:q3,list:V3,paragraph:G3,root:K3,strong:Y3,table:X3,tableCell:Q3,tableRow:W3,text:J3,thematicBreak:eB,toml:kc,yaml:kc,definition:kc,footnoteDefinition:kc};function kc(){}const ZC=-1,qf=0,El=1,tf=2,Pg=3,Ug=4,Hg=5,qg=6,JC=7,ek=8,I1=typeof self=="object"?self:globalThis,nB=(e,t)=>{const n=(o,l)=>(e.set(l,o),o),i=o=>{if(e.has(o))return e.get(o);const[l,u]=t[o];switch(l){case qf:case ZC:return n(u,o);case El:{const f=n([],o);for(const d of u)f.push(i(d));return f}case tf:{const f=n({},o);for(const[d,p]of u)f[i(d)]=i(p);return f}case Pg:return n(new Date(u),o);case Ug:{const{source:f,flags:d}=u;return n(new RegExp(f,d),o)}case Hg:{const f=n(new Map,o);for(const[d,p]of u)f.set(i(d),i(p));return f}case qg:{const f=n(new Set,o);for(const d of u)f.add(i(d));return f}case JC:{const{name:f,message:d}=u;return n(new I1[f](d),o)}case ek:return n(BigInt(u),o);case"BigInt":return n(Object(BigInt(u)),o);case"ArrayBuffer":return n(new Uint8Array(u).buffer,u);case"DataView":{const{buffer:f}=new Uint8Array(u);return n(new DataView(f),u)}}return n(new I1[l](u),o)};return i},B1=e=>nB(new Map,e)(0),To="",{toString:rB}={},{keys:iB}=Object,sl=e=>{const t=typeof e;if(t!=="object"||!e)return[qf,t];const n=rB.call(e).slice(8,-1);switch(n){case"Array":return[El,To];case"Object":return[tf,To];case"Date":return[Pg,To];case"RegExp":return[Ug,To];case"Map":return[Hg,To];case"Set":return[qg,To];case"DataView":return[El,n]}return n.includes("Array")?[El,n]:n.includes("Error")?[JC,n]:[tf,n]},Ec=([e,t])=>e===qf&&(t==="function"||t==="symbol"),aB=(e,t,n,i)=>{const o=(u,f)=>{const d=i.push(u)-1;return n.set(f,d),d},l=u=>{if(n.has(u))return n.get(u);let[f,d]=sl(u);switch(f){case qf:{let m=u;switch(d){case"bigint":f=ek,m=u.toString();break;case"function":case"symbol":if(e)throw new TypeError("unable to serialize "+d);m=null;break;case"undefined":return o([ZC],u)}return o([f,m],u)}case El:{if(d){let v=u;return d==="DataView"?v=new Uint8Array(u.buffer):d==="ArrayBuffer"&&(v=new Uint8Array(u)),o([d,[...v]],u)}const m=[],g=o([f,m],u);for(const v of u)m.push(l(v));return g}case tf:{if(d)switch(d){case"BigInt":return o([d,u.toString()],u);case"Boolean":case"Number":case"String":return o([d,u.valueOf()],u)}if(t&&"toJSON"in u)return l(u.toJSON());const m=[],g=o([f,m],u);for(const v of iB(u))(e||!Ec(sl(u[v])))&&m.push([l(v),l(u[v])]);return g}case Pg:return o([f,u.toISOString()],u);case Ug:{const{source:m,flags:g}=u;return o([f,{source:m,flags:g}],u)}case Hg:{const m=[],g=o([f,m],u);for(const[v,b]of u)(e||!(Ec(sl(v))||Ec(sl(b))))&&m.push([l(v),l(b)]);return g}case qg:{const m=[],g=o([f,m],u);for(const v of u)(e||!Ec(sl(v)))&&m.push(l(v));return g}}const{message:p}=u;return o([f,{name:d,message:p}],u)};return l},j1=(e,{json:t,lossy:n}={})=>{const i=[];return aB(!(t||n),!!t,new Map,i)(e),i},nf=typeof structuredClone=="function"?(e,t)=>t&&("json"in t||"lossy"in t)?B1(j1(e,t)):structuredClone(e):(e,t)=>B1(j1(e,t));function oB(e,t){const n=[{type:"text",value:"↩"}];return t>1&&n.push({type:"element",tagName:"sup",properties:{},children:[{type:"text",value:String(t)}]}),n}function sB(e,t){return"Back to reference "+(e+1)+(t>1?"-"+t:"")}function lB(e){const t=typeof e.options.clobberPrefix=="string"?e.options.clobberPrefix:"user-content-",n=e.options.footnoteBackContent||oB,i=e.options.footnoteBackLabel||sB,o=e.options.footnoteLabel||"Footnotes",l=e.options.footnoteLabelTagName||"h2",u=e.options.footnoteLabelProperties||{className:["sr-only"]},f=[];let d=-1;for(;++d0&&w.push({type:"text",value:" "});let O=typeof n=="string"?n:n(d,b);typeof O=="string"&&(O={type:"text",value:O}),w.push({type:"element",tagName:"a",properties:{href:"#"+t+"fnref-"+v+(b>1?"-"+b:""),dataFootnoteBackref:"",ariaLabel:typeof i=="string"?i:i(d,b),className:["data-footnote-backref"]},children:Array.isArray(O)?O:[O]})}const C=m[m.length-1];if(C&&C.type==="element"&&C.tagName==="p"){const O=C.children[C.children.length-1];O&&O.type==="text"?O.value+=" ":C.children.push({type:"text",value:" "}),C.children.push(...w)}else m.push(...w);const A={type:"element",tagName:"li",properties:{id:t+"fn-"+v},children:e.wrap(m,!0)};e.patch(p,A),f.push(A)}if(f.length!==0)return{type:"element",tagName:"section",properties:{dataFootnotes:!0,className:["footnotes"]},children:[{type:"element",tagName:l,properties:{...nf(u),id:"footnote-label"},children:[{type:"text",value:o}]},{type:"text",value:` +`},{type:"element",tagName:"ol",properties:{},children:e.wrap(f,!0)},{type:"text",value:` +`}]}}const tk=function(e){if(e==null)return dB;if(typeof e=="function")return Ff(e);if(typeof e=="object")return Array.isArray(e)?uB(e):cB(e);if(typeof e=="string")return fB(e);throw new Error("Expected function, string, or object as test")};function uB(e){const t=[];let n=-1;for(;++n":""))+")"})}return v;function v(){let b=nk,w,S,C;if((!t||l(d,p,m[m.length-1]||void 0))&&(b=yB(n(d,m)),b[0]===$1))return b;if("children"in d&&d.children){const A=d;if(A.children&&b[0]!==mB)for(S=(i?A.children.length:-1)+u,C=m.concat(A);S>-1&&S0&&n.push({type:"text",value:` +`}),n}function P1(e){let t=0,n=e.charCodeAt(t);for(;n===9||n===32;)t++,n=e.charCodeAt(t);return e.slice(t)}function U1(e,t){const n=vB(e,t),i=n.one(e,void 0),o=lB(n),l=Array.isArray(i)?{type:"root",children:i}:i||{type:"root",children:[]};return o&&l.children.push({type:"text",value:` +`},o),l}function kB(e,t){return e&&"run"in e?async function(n,i){const o=U1(n,{file:i,...t});await e.run(o,i)}:function(n,i){return U1(n,{file:i,...e||t})}}function H1(e){if(e)throw e}var Dh,q1;function EB(){if(q1)return Dh;q1=1;var e=Object.prototype.hasOwnProperty,t=Object.prototype.toString,n=Object.defineProperty,i=Object.getOwnPropertyDescriptor,o=function(p){return typeof Array.isArray=="function"?Array.isArray(p):t.call(p)==="[object Array]"},l=function(p){if(!p||t.call(p)!=="[object Object]")return!1;var m=e.call(p,"constructor"),g=p.constructor&&p.constructor.prototype&&e.call(p.constructor.prototype,"isPrototypeOf");if(p.constructor&&!m&&!g)return!1;var v;for(v in p);return typeof v>"u"||e.call(p,v)},u=function(p,m){n&&m.name==="__proto__"?n(p,m.name,{enumerable:!0,configurable:!0,value:m.newValue,writable:!0}):p[m.name]=m.newValue},f=function(p,m){if(m==="__proto__")if(e.call(p,m)){if(i)return i(p,m).value}else return;return p[m]};return Dh=function d(){var p,m,g,v,b,w,S=arguments[0],C=1,A=arguments.length,O=!1;for(typeof S=="boolean"&&(O=S,S=arguments[1]||{},C=2),(S==null||typeof S!="object"&&typeof S!="function")&&(S={});Cu.length;let d;f&&u.push(o);try{d=e.apply(this,u)}catch(p){const m=p;if(f&&n)throw m;return o(m)}f||(d&&d.then&&typeof d.then=="function"?d.then(l,o):d instanceof Error?o(d):l(d))}function o(u,...f){n||(n=!0,t(u,...f))}function l(u){o(null,u)}}const Br={basename:RB,dirname:OB,extname:MB,join:NB,sep:"/"};function RB(e,t){if(t!==void 0&&typeof t!="string")throw new TypeError('"ext" argument must be a string');ru(e);let n=0,i=-1,o=e.length,l;if(t===void 0||t.length===0||t.length>e.length){for(;o--;)if(e.codePointAt(o)===47){if(l){n=o+1;break}}else i<0&&(l=!0,i=o+1);return i<0?"":e.slice(n,i)}if(t===e)return"";let u=-1,f=t.length-1;for(;o--;)if(e.codePointAt(o)===47){if(l){n=o+1;break}}else u<0&&(l=!0,u=o+1),f>-1&&(e.codePointAt(o)===t.codePointAt(f--)?f<0&&(i=o):(f=-1,i=u));return n===i?i=u:i<0&&(i=e.length),e.slice(n,i)}function OB(e){if(ru(e),e.length===0)return".";let t=-1,n=e.length,i;for(;--n;)if(e.codePointAt(n)===47){if(i){t=n;break}}else i||(i=!0);return t<0?e.codePointAt(0)===47?"/":".":t===1&&e.codePointAt(0)===47?"//":e.slice(0,t)}function MB(e){ru(e);let t=e.length,n=-1,i=0,o=-1,l=0,u;for(;t--;){const f=e.codePointAt(t);if(f===47){if(u){i=t+1;break}continue}n<0&&(u=!0,n=t+1),f===46?o<0?o=t:l!==1&&(l=1):o>-1&&(l=-1)}return o<0||n<0||l===0||l===1&&o===n-1&&o===i+1?"":e.slice(o,n)}function NB(...e){let t=-1,n;for(;++t0&&e.codePointAt(e.length-1)===47&&(n+="/"),t?"/"+n:n}function LB(e,t){let n="",i=0,o=-1,l=0,u=-1,f,d;for(;++u<=e.length;){if(u2){if(d=n.lastIndexOf("/"),d!==n.length-1){d<0?(n="",i=0):(n=n.slice(0,d),i=n.length-1-n.lastIndexOf("/")),o=u,l=0;continue}}else if(n.length>0){n="",i=0,o=u,l=0;continue}}t&&(n=n.length>0?n+"/..":"..",i=2)}else n.length>0?n+="/"+e.slice(o+1,u):n=e.slice(o+1,u),i=u-o-1;o=u,l=0}else f===46&&l>-1?l++:l=-1}return n}function ru(e){if(typeof e!="string")throw new TypeError("Path must be a string. Received "+JSON.stringify(e))}const zB={cwd:IB};function IB(){return"/"}function _m(e){return!!(e!==null&&typeof e=="object"&&"href"in e&&e.href&&"protocol"in e&&e.protocol&&e.auth===void 0)}function BB(e){if(typeof e=="string")e=new URL(e);else if(!_m(e)){const t=new TypeError('The "path" argument must be of type string or an instance of URL. Received `'+e+"`");throw t.code="ERR_INVALID_ARG_TYPE",t}if(e.protocol!=="file:"){const t=new TypeError("The URL must be of scheme file");throw t.code="ERR_INVALID_URL_SCHEME",t}return jB(e)}function jB(e){if(e.hostname!==""){const i=new TypeError('File URL host must be "localhost" or empty on darwin');throw i.code="ERR_INVALID_FILE_URL_HOST",i}const t=e.pathname;let n=-1;for(;++n0){let[b,...w]=m;const S=i[v][1];Am(S)&&Am(b)&&(b=Lh(!0,S,b)),i[v]=[p,b,...w]}}}}const HB=new Fg().freeze();function jh(e,t){if(typeof t!="function")throw new TypeError("Cannot `"+e+"` without `parser`")}function $h(e,t){if(typeof t!="function")throw new TypeError("Cannot `"+e+"` without `compiler`")}function Ph(e,t){if(t)throw new Error("Cannot call `"+e+"` on a frozen processor.\nCreate a new processor first, by calling it: use `processor()` instead of `processor`.")}function V1(e){if(!Am(e)||typeof e.type!="string")throw new TypeError("Expected node, got `"+e+"`")}function G1(e,t,n){if(!n)throw new Error("`"+e+"` finished async. Use `"+t+"` instead")}function Tc(e){return qB(e)?e:new ik(e)}function qB(e){return!!(e&&typeof e=="object"&&"message"in e&&"messages"in e)}function FB(e){return typeof e=="string"||VB(e)}function VB(e){return!!(e&&typeof e=="object"&&"byteLength"in e&&"byteOffset"in e)}const GB="https://github.com/remarkjs/react-markdown/blob/main/changelog.md",K1=[],Y1={allowDangerousHtml:!0},KB=/^(https?|ircs?|mailto|xmpp)$/i,YB=[{from:"astPlugins",id:"remove-buggy-html-in-markdown-parser"},{from:"allowDangerousHtml",id:"remove-buggy-html-in-markdown-parser"},{from:"allowNode",id:"replace-allownode-allowedtypes-and-disallowedtypes",to:"allowElement"},{from:"allowedTypes",id:"replace-allownode-allowedtypes-and-disallowedtypes",to:"allowedElements"},{from:"className",id:"remove-classname"},{from:"disallowedTypes",id:"replace-allownode-allowedtypes-and-disallowedtypes",to:"disallowedElements"},{from:"escapeHtml",id:"remove-buggy-html-in-markdown-parser"},{from:"includeElementIndex",id:"#remove-includeelementindex"},{from:"includeNodeIndex",id:"change-includenodeindex-to-includeelementindex"},{from:"linkTarget",id:"remove-linktarget"},{from:"plugins",id:"change-plugins-to-remarkplugins",to:"remarkPlugins"},{from:"rawSourcePos",id:"#remove-rawsourcepos"},{from:"renderers",id:"change-renderers-to-components",to:"components"},{from:"source",id:"change-source-to-children",to:"children"},{from:"sourcePos",id:"#remove-sourcepos"},{from:"transformImageUri",id:"#add-urltransform",to:"urlTransform"},{from:"transformLinkUri",id:"#add-urltransform",to:"urlTransform"}];function XB(e){const t=WB(e),n=QB(e);return ZB(t.runSync(t.parse(n),n),e)}function WB(e){const t=e.rehypePlugins||K1,n=e.remarkPlugins||K1,i=e.remarkRehypeOptions?{...e.remarkRehypeOptions,...Y1}:Y1;return HB().use(R3).use(n).use(kB,i).use(t)}function QB(e){const t=e.children||"",n=new ik;return typeof t=="string"&&(n.value=t),n}function ZB(e,t){const n=t.allowedElements,i=t.allowElement,o=t.components,l=t.disallowedElements,u=t.skipHtml,f=t.unwrapDisallowed,d=t.urlTransform||JB;for(const m of YB)Object.hasOwn(t,m.from)&&(""+m.from+(m.to?"use `"+m.to+"` instead":"remove it")+GB+m.id,void 0);return rk(e,p),s4(e,{Fragment:B.Fragment,components:o,ignoreInvalidStyle:!0,jsx:B.jsx,jsxs:B.jsxs,passKeys:!0,passNode:!0});function p(m,g,v){if(m.type==="raw"&&v&&typeof g=="number")return u?v.children.splice(g,1):v.children[g]={type:"text",value:m.value},g;if(m.type==="element"){let b;for(b in Oh)if(Object.hasOwn(Oh,b)&&Object.hasOwn(m.properties,b)){const w=m.properties[b],S=Oh[b];(S===null||S.includes(m.tagName))&&(m.properties[b]=d(String(w||""),b,m))}}if(m.type==="element"){let b=n?!n.includes(m.tagName):l?l.includes(m.tagName):!1;if(!b&&i&&typeof g=="number"&&(b=!i(m,g,v)),b&&v&&typeof g=="number")return f&&m.children?v.children.splice(g,1,...m.children):v.children.splice(g,1),g}}}function JB(e){const t=e.indexOf(":"),n=e.indexOf("?"),i=e.indexOf("#"),o=e.indexOf("/");return t===-1||o!==-1&&t>o||n!==-1&&t>n||i!==-1&&t>i||KB.test(e.slice(0,t))?e:""}const e6=bi(B.jsx("path",{d:"M11 18c0 1.1-.9 2-2 2s-2-.9-2-2 .9-2 2-2 2 .9 2 2m-2-8c-1.1 0-2 .9-2 2s.9 2 2 2 2-.9 2-2-.9-2-2-2m0-6c-1.1 0-2 .9-2 2s.9 2 2 2 2-.9 2-2-.9-2-2-2m6 4c1.1 0 2-.9 2-2s-.9-2-2-2-2 .9-2 2 .9 2 2 2m0 2c-1.1 0-2 .9-2 2s.9 2 2 2 2-.9 2-2-.9-2-2-2m0 6c-1.1 0-2 .9-2 2s.9 2 2 2 2-.9 2-2-.9-2-2-2"}),"DragIndicator"),t6=bi(B.jsx("path",{d:"M12 4.5C7 4.5 2.73 7.61 1 12c1.73 4.39 6 7.5 11 7.5s9.27-3.11 11-7.5c-1.73-4.39-6-7.5-11-7.5M12 17c-2.76 0-5-2.24-5-5s2.24-5 5-5 5 2.24 5 5-2.24 5-5 5m0-8c-1.66 0-3 1.34-3 3s1.34 3 3 3 3-1.34 3-3-1.34-3-3-3"}),"Visibility"),n6=bi(B.jsx("path",{d:"M12 7c2.76 0 5 2.24 5 5 0 .65-.13 1.26-.36 1.83l2.92 2.92c1.51-1.26 2.7-2.89 3.43-4.75-1.73-4.39-6-7.5-11-7.5-1.4 0-2.74.25-3.98.7l2.16 2.16C10.74 7.13 11.35 7 12 7M2 4.27l2.28 2.28.46.46C3.08 8.3 1.78 10.02 1 12c1.73 4.39 6 7.5 11 7.5 1.55 0 3.03-.3 4.38-.84l.42.42L19.73 22 21 20.73 3.27 3zM7.53 9.8l1.55 1.55c-.05.21-.08.43-.08.65 0 1.66 1.34 3 3 3 .22 0 .44-.03.65-.08l1.55 1.55c-.67.33-1.41.53-2.2.53-2.76 0-5-2.24-5-5 0-.79.2-1.53.53-2.2m4.31-.78 3.15 3.15.02-.16c0-1.66-1.34-3-3-3z"}),"VisibilityOff"),r6=bi(B.jsx("path",{d:"M11 18h2v-2h-2zm1-16C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2m0 18c-4.41 0-8-3.59-8-8s3.59-8 8-8 8 3.59 8 8-3.59 8-8 8m0-14c-2.21 0-4 1.79-4 4h2c0-1.1.9-2 2-2s2 .9 2 2c0 2-3 1.75-3 5h2c0-2.25 3-2.5 3-5 0-2.21-1.79-4-4-4"}),"HelpOutline");String.prototype.capitalize=function(){return this.charAt(0).toUpperCase()+this.slice(1)};const i6=({type:e,module:t,toggleModule:n,enabledModules:i,configValues:o})=>{const{attributes:l,listeners:u,setNodeRef:f,transform:d,transition:p,isDragging:m}=yL({id:t.name}),g={...wS.style,transform:Xi.Transform.toString(d),transition:p,zIndex:m?"100":"auto",opacity:m?.3:1};let v=t.name;const[b,w]=T.useState(!1),[S,C]=T.useState(!1),A=i[e].find(O=>O[0]===v)[1];return B.jsxs(sx,{ref:f,size:{xs:6,sm:4,md:3},style:g,children:[B.jsxs(wS,{children:[B.jsx(iM,{title:B.jsx(ax,{style:{paddingRight:"0 !important"},control:B.jsx(Qw,{title:"Check to enable this module",sx:{paddingTop:0,paddingBottom:0},id:v,onClick:n,checked:A}),label:t.display_name})}),B.jsx(QO,{children:B.jsxs(Pr,{sx:{justifyContent:"space-between",display:"flex",width:"100%"},children:[B.jsxs(Pr,{children:[B.jsx(rm,{title:"Module information",size:"small",onClick:()=>w(!0),children:B.jsx(r6,{})}),A&&t.configs&&v!="cli_feeder"?B.jsx(om,{size:"small",onClick:()=>C(!0),children:"Configure"}):null]}),B.jsx(rm,{size:"small",title:"Drag to reorder",sx:{cursor:"grab"},...u,...l,children:B.jsx(e6,{})})]})})]}),B.jsxs(tx,{open:b,onClose:()=>w(!1),maxWidth:"lg",children:[B.jsx(rx,{children:t.display_name}),B.jsx(nx,{children:B.jsx(XB,{children:t.manifest.description.split(` +`).map(O=>O.trim()).join(` +`)})})]}),t.configs&&v!="cli_feeder"&&B.jsx(o6,{module:t,open:S,setOpen:C,configValues:o})]})};function a6({config_value:e,module:t,configValues:n}){const[i,o]=T.useState(!1),l=()=>o(C=>!C),u=C=>{C.preventDefault()},f=C=>{C.preventDefault()};function d(C,A){n[t.name][C]=A}const p=t.configs[e],g=e.replace(/_/g," ").capitalize(),v=n[t.name][e]||p.default,b=e.toLowerCase(),w=b.includes("password")||b.includes("secret")||b.includes("token")||b.includes("key")||b.includes("api_hash")||p.type==="password",S=w?"password":p.type==="int"?"number":"text";return B.jsxs(Pr,{children:[B.jsxs(en,{variant:"body1",style:{fontWeight:"bold"},children:[g," ",p.required&&"(required)"," "]}),B.jsxs(ix,{size:"small",children:[p.type==="bool"?B.jsx(ax,{control:B.jsx(Qw,{defaultChecked:v,size:"small",id:`${t}.${e}`,onChange:C=>{d(e,C.target.checked)}}),label:p.help.capitalize()}):p.choices!==void 0?B.jsx(sg,{size:"small",id:`${t}.${e}`,defaultValue:p.default,value:v,onChange:C=>{d(e,C.target.value)},children:p.choices.map(C=>B.jsx(tD,{value:C,children:C},`${t}.${e}.${C}`))}):p.type==="json_loader"?B.jsx($S,{multiline:!0,size:"small",id:`${t}.${e}`,defaultValue:JSON.stringify(v,null,2),rows:6,onChange:C=>{try{let A=JSON.parse(C.target.value);d(e,A)}catch(A){console.log(A)}}}):B.jsx($S,{size:"small",id:`${t}.${e}`,defaultValue:v,type:i?"text":S,onChange:C=>{d(e,C.target.value)},required:p.required,slotProps:w?{input:{endAdornment:B.jsx(RN,{position:"end",children:B.jsx(rm,{"aria-label":"toggle password visibility",onClick:l,onMouseDown:u,onMouseUp:f,children:i?B.jsx(n6,{}):B.jsx(t6,{})})})}}:{}}),p.type!=="bool"&&B.jsx(ox,{children:p.help.capitalize()})]})]})}function o6({module:e,open:t,setOpen:n,configValues:i}){return B.jsx(B.Fragment,{children:B.jsxs(tx,{open:t,onClose:()=>n(!1),maxWidth:"lg",children:[B.jsx(rx,{children:e.display_name}),B.jsx(nx,{children:B.jsx(px,{direction:"column",spacing:1,children:Object.keys(e.configs).map(o=>B.jsx(a6,{config_value:o,module:e,configValues:i},o))})})]})})}function s6({setYamlFile:e}){const[t,n]=T.useState(!1),[i,o]=T.useState(B.jsx(B.Fragment,{children:"Drag and drop your orchestration.yaml file here, or click to select a file."}));T.useRef(null);function l(u){let f=u.target.files[0];if(f.type.indexOf("yaml")===-1){n(!0),o(B.jsx(B.Fragment,{children:"Invalid type, only YAML files are accepted."}));return}let d=new FileReader;d.onload=function(p){let m=p.target?p.target.result:"";try{let g=xC(m);if(g.errors.length>0){n(!0),o(B.jsx(B.Fragment,{children:"Invalid file. Make sure your Orchestration is a valid YAML file with a 'steps' section in it."}));return}else n(!1),o(B.jsx(B.Fragment,{children:"File loaded successfully."}));let v=g.get("steps");if(!v){n(!0),o(B.jsx(B.Fragment,{children:"Invalid file. Your orchestration file must have a 'steps' section in it."}));return}const b={feeder:"feeders",formatter:"formatters",archivers:"extractors"};let w=!1;for(let S of Object.keys(b))if(v.get(S)!==void 0){n(!0),o(B.jsxs(B.Fragment,{children:["Invalid file. Your orchestration file appears to be in the old (v0.12) format with a '",S,"' section.",B.jsx("br",{}),"You should manually update your orchestration file first (hint: ",S," → ",b[S],")"]})),w=!0;return}e(g)}catch(g){console.error(g)}},d.readAsText(f)}return B.jsx(B.Fragment,{children:B.jsxs("div",{style:{position:"relative",width:"100%",border:"dashed",borderRadius:"5px",textAlign:"center",borderWidth:"1px",padding:"20px"},onDragEnter:u=>{u.currentTarget.style.backgroundColor="var(--mui-palette-LinearProgress-infoBg)"},onDragLeave:u=>{u.currentTarget.style.backgroundColor=""},onDrop:u=>{u.currentTarget.style.backgroundColor=""},children:[B.jsx(_D,{style:{fontSize:50}}),B.jsx("input",{style:{opacity:0,position:"absolute",top:0,left:0,width:"100%",height:"100%",cursor:"pointer"},type:"file",id:"file",accept:".yaml",onChange:l}),B.jsx(en,{variant:"body1",color:t?"error":"",children:i})]})})}function l6({stepType:e,setEnabledModules:t,enabledModules:n,configValues:i}){const[o,l]=T.useState(!1),[u,f]=T.useState(),[d,p]=T.useState([]);T.useEffect(()=>{p(n[e].map(([w,S])=>w))},[n]);const m=w=>{let S=w.target.id,C=w.target.checked;(e==="feeders"||e==="formatters")&&n[e].filter(([_,M])=>_!==S&&M||C&&_===S).length>1?l(!0):l(!1);let A={...n};A[e]=n[e].map(([O,_])=>O===S?[O,C]:[O,_]),t(A)},g=qD(US(pg),US(fg,{coordinateGetter:SL})),v=w=>{f(w.active.id)},b=w=>{f(void 0);const{active:S,over:C}=w;if(S.id!==(C==null?void 0:C.id)){const A=d.indexOf(S.id),O=d.indexOf(C==null?void 0:C.id);let _=mg(d,A,O),M={...n};M[e]=n[e].sort((R,D)=>_.indexOf(R[0])-_.indexOf(D[0])),t(M)}};return B.jsxs(B.Fragment,{children:[B.jsxs(Pr,{sx:{my:4},children:[B.jsx(en,{id:e,variant:"h6",style:{textTransform:"capitalize"},children:e}),B.jsxs(en,{variant:"body1",children:["Select the ",B.jsx("a",{href:"{let S=dl[w];return B.jsx(i6,{type:e,module:S,toggleModule:m,enabledModules:n,configValues:i},w)}),B.jsx(oL,{children:u?B.jsx("div",{style:{width:"100%",height:"100%",backgroundColor:"grey",opacity:.1}}):null})]})},e)})]})}function u6(){const[e,t]=T.useState(new eu),[n,i]=T.useState(Object.fromEntries(Object.keys(il).map(f=>[f,il[f].map(d=>[d,!1])]))),[o,l]=T.useState(Object.keys(dl).reduce((f,d)=>(f[d]={},f),{})),u=function(f=!1){let d=n,p=null;if(!e||e.contents==null?p=xC(xL):p=e,wh.forEach(m=>{let g=m+"s",v=p.getIn(["steps",g]);d[g].forEach(([b,w])=>{var A,O;let S=v.items.findIndex(_=>(_.value||_)===b),C=p.getIn(["steps",g],!0);w&&S===-1?(p.addIn(["steps",g],b),C.commentBefore=(A=C.commentBefore)==null?void 0:A.replace(` + - `+b,""),C.comment=(O=C.comment)==null?void 0:O.replace(` + - `+b,"")):!w&&S!==-1&&(p.deleteIn(["steps",g,S]),C.commentBefore+=` + - `+b,p.setIn(["steps",g],C))}),v.items.sort((b,w)=>d[g].findIndex(S=>S[0]===(b.value||b))-d[g].findIndex(S=>S[0]===(w.value||w))),v.flow=!v.items.length}),Object.keys(o).forEach(m=>{let g=p.get(m,!0);g?(Object.keys(o[m]).forEach(v=>{let b=g.get(v,!0);b?(b.value=o[m][v],g.set(v,b)):g.set(v,o[m][v])}),p.set(m,g)):o[m]&&Object.keys(o[m]).length>0&&p.set(m,o[m])}),f)navigator.clipboard.writeText(String(p)).then(()=>{alert("Settings copied to clipboard.")});else{const m=new Blob([String(p)],{type:"application/x-yaml"}),g=URL.createObjectURL(m),v=document.createElement("a");v.href=g,v.download="orchestration.yaml",v.click()}};return T.useEffect(()=>{let f={};Object.keys(dl).map(d=>{let m=dl[d].configs;m&&(f[d]={},Object.keys(m).map(g=>{let v=m[g];v.default!==void 0&&(f[d][g]=v.default)}))}),l(f)},[]),T.useEffect(()=>{if(!e||e.contents==null)return;let f=e.toJS(),d=f.steps,p=Object.fromEntries(Object.keys(il).map(g=>[g,il[g].map(v=>[v,d[g].indexOf(v)!==-1]).sort((v,b)=>{let w=d[g].indexOf(v[0]),S=d[g].indexOf(b[0]);return w===-1&&S===-1?v-b:S===-1?-1:w===-1?1:w-S})]).sort((g,v)=>wh.indexOf(g[0])-wh.indexOf(v[0])));i(p);let m=f;delete m.steps,l(Object.keys(dl).reduce((g,v)=>(g[v]=m[v]||{},g),{}))},[e]),B.jsx(vM,{maxWidth:"lg",children:B.jsxs(Pr,{sx:{my:4},children:[B.jsxs(Pr,{sx:{my:4},children:[B.jsx(en,{variant:"h5",children:"1. Select your orchestration.yaml settings file."}),B.jsx(en,{variant:"body1",children:"Or skip this step to start from scratch"}),B.jsx(s6,{setYamlFile:t})]}),B.jsxs(Pr,{sx:{my:4},children:[B.jsx(en,{variant:"h5",children:"2. Choose the Modules you wish to enable/disable"}),Object.keys(il).map(f=>B.jsx(Pr,{sx:{my:4},children:B.jsx(l6,{stepType:f,setEnabledModules:i,enabledModules:n,configValues:o})},f))]}),B.jsxs(Pr,{sx:{my:4},children:[B.jsx(en,{variant:"h5",children:"3. Configure your Enabled Modules"}),B.jsx(en,{variant:"body1",children:"Next to each module you've enabled, you can click 'Configure' to set the module's settings."})]}),B.jsxs(Pr,{sx:{my:4},children:[B.jsx(en,{variant:"h5",children:"4. Save your settings"}),B.jsxs(px,{direction:"row",spacing:2,sx:{my:2},children:[B.jsx(om,{variant:"contained",color:"primary",onClick:()=>u(!0),children:"Copy Settings to Clipboard"}),B.jsx(om,{variant:"contained",color:"primary",onClick:()=>u(),children:"Save Settings to File"})]})]})]})})}function c6(){const[e,t]=T.useState("light");T.useEffect(()=>{t(window.localStorage.getItem("theme")||"light")},[]);var n=new MutationObserver(function(o){t(window.localStorage.getItem("theme")||"light")});n.observe(document.documentElement,{attributes:!0,attributeFilter:["data-theme"]});const i=vf({palette:{mode:e=="light"?"light":"dark"},cssVariables:!0});return B.jsxs(kR,{theme:i,children:[B.jsx(kM,{}),B.jsx(u6,{})]})}E2.createRoot(document.getElementById("root")).render(B.jsx(T.StrictMode,{children:B.jsx(c6,{})}));
diff --git a/docs/source/installation/setup.md b/docs/source/installation/setup.md index f5b6e9d..10691ee 100644 --- a/docs/source/installation/setup.md +++ b/docs/source/installation/setup.md @@ -1,7 +1,6 @@ # Getting Started ```{toctree} -:maxdepth: 1 :hidden: installation.md @@ -9,6 +8,7 @@ configurations.md config_editor.md authentication.md requirements.md +faq.md config_cheatsheet.md ``` @@ -27,17 +27,18 @@ The way you run the Auto Archiver depends on how you installed it (docker instal If you installed Auto Archiver using docker, open up your terminal, and copy-paste / type the following command: ```bash -docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver +docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver ``` breaking this command down: 1. `docker run` tells docker to start a new container (an instance of the image) - 2. `--rm` makes sure this container is removed after execution (less garbage locally) - 3. `-v $PWD/secrets:/app/secrets` - your secrets folder with settings + 2. `-it` tells docker to run in 'interactive mode' so that we get nice colour logs + 3. `--rm` makes sure this container is removed after execution (less garbage locally) + 4. `-v $PWD/secrets:/app/secrets` - your secrets folder with settings 1. `-v` is a volume flag which means a folder that you have on your computer will be connected to a folder inside the docker container 2. `$PWD/secrets` points to a `secrets/` folder in your current working directory (where your console points to), we use this folder as a best practice to hold all the secrets/tokens/passwords/... you use 3. `/app/secrets` points to the path the docker container where this image can be found - 4. `-v $PWD/local_archive:/app/local_archive` - (optional) if you use local_storage + 5. `-v $PWD/local_archive:/app/local_archive` - (optional) if you use local_storage 1. `-v` same as above, this is a volume instruction 2. `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker 3. `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file @@ -48,14 +49,14 @@ The invocations below will run the auto-archiver Docker image using a configurat ```bash # Have auto-archiver run with the default settings, generating a settings file in ./secrets/orchestration.yaml -docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver +docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver # uses the same configuration, but with the `gsheet_feeder`, a header on row 2 and with some different column names # Note this expects you to have followed the [Google Sheets setup](how_to/google_sheets.md) and added your service_account.json to the `secrets/` folder # notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided -docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --feeders=gsheet_feeder --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}' +docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --feeders=gsheet_feeder --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}' # Runs auto-archiver for the first time, but in 'full' mode, enabling all modules to get a full settings file -docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --mode full +docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --mode full ``` ------------ diff --git a/docs/source/installation/upgrading.md b/docs/source/installation/upgrading.md new file mode 100644 index 0000000..3c77dd8 --- /dev/null +++ b/docs/source/installation/upgrading.md @@ -0,0 +1,30 @@ + +# Upgrading + +If an update is available, then you will see a message in the logs when you +run Auto Archiver. Here's what those logs look like: + +```{code} bash +********* IMPORTANT: UPDATE AVAILABLE ******** +A new version of auto-archiver is available (v0.13.6, you have 0.13.4) +Make sure to update to the latest version using: `pip install --upgrade auto-archiver` +``` + +Upgrading Auto Archiver depends on the way you installed it. + +## Docker + +To upgrade using docker, update the docker image with: + +``` +docker pull bellingcat/auto-archiver:latest +``` + +## Pip + +To upgrade the pip package, use: + +``` +pip install --upgrade auto-archiver +``` + diff --git a/scripts/generate_settings_schema.py b/scripts/generate_settings_schema.py index fa7aaf6..d1727f4 100644 --- a/scripts/generate_settings_schema.py +++ b/scripts/generate_settings_schema.py @@ -59,4 +59,5 @@ output_schema = { current_file_dir = os.path.dirname(os.path.abspath(__file__)) output_file = os.path.join(current_file_dir, "settings/src/schema.json") with open(output_file, "w") as file: + print(f"Writing schema to {output_file}") json.dump(output_schema, file, indent=4, cls=SchemaEncoder) diff --git a/scripts/settings/package-lock.json b/scripts/settings/package-lock.json index cd40c14..88d4d33 100644 --- a/scripts/settings/package-lock.json +++ b/scripts/settings/package-lock.json @@ -12,7 +12,7 @@ "@dnd-kit/sortable": "^10.0.0", "@emotion/react": "latest", "@emotion/styled": "latest", - "@mui/icons-material": "latest", + "@mui/icons-material": "^6.4.7", "@mui/material": "latest", "react": "19.0.0", "react-dom": "19.0.0", @@ -997,9 +997,9 @@ } }, "node_modules/@mui/core-downloads-tracker": { - "version": "6.4.6", - "resolved": "https://registry.npmjs.org/@mui/core-downloads-tracker/-/core-downloads-tracker-6.4.6.tgz", - "integrity": "sha512-rho5Q4IscbrVmK9rCrLTJmjLjfH6m/NcqKr/mchvck0EIXlyYUB9+Z0oVmkt/+Mben43LMRYBH8q/Uzxj/c4Vw==", + "version": "6.4.7", + "resolved": "https://registry.npmjs.org/@mui/core-downloads-tracker/-/core-downloads-tracker-6.4.7.tgz", + "integrity": "sha512-XjJrKFNt9zAKvcnoIIBquXyFyhfrHYuttqMsoDS7lM7VwufYG4fAPw4kINjBFg++fqXM2BNAuWR9J7XVIuKIKg==", "license": "MIT", "funding": { "type": "opencollective", @@ -1007,9 +1007,9 @@ } }, "node_modules/@mui/icons-material": { - "version": "6.4.6", - "resolved": "https://registry.npmjs.org/@mui/icons-material/-/icons-material-6.4.6.tgz", - "integrity": "sha512-rGJBvIQQbQAlyKYljHQ8wAQS/K2/uYwvemcpygnAmCizmCI4zSF9HQPuiG8Ql4YLZ6V/uKjA3WHIYmF/8sV+pQ==", + "version": "6.4.7", + "resolved": "https://registry.npmjs.org/@mui/icons-material/-/icons-material-6.4.7.tgz", + "integrity": "sha512-Rk8cs9ufQoLBw582Rdqq7fnSXXZTqhYRbpe1Y5SAz9lJKZP3CIdrj0PfG8HJLGw1hrsHFN/rkkm70IDzhJsG1g==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.26.0" @@ -1022,7 +1022,7 @@ "url": "https://opencollective.com/mui-org" }, "peerDependencies": { - "@mui/material": "^6.4.6", + "@mui/material": "^6.4.7", "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", "react": "^17.0.0 || ^18.0.0 || ^19.0.0" }, @@ -1033,14 +1033,14 @@ } }, "node_modules/@mui/material": { - "version": "6.4.6", - "resolved": "https://registry.npmjs.org/@mui/material/-/material-6.4.6.tgz", - "integrity": "sha512-6UyAju+DBOdMogfYmLiT3Nu7RgliorimNBny1pN/acOjc+THNFVE7hlxLyn3RDONoZJNDi/8vO4AQQr6dLAXqA==", + "version": "6.4.7", + "resolved": "https://registry.npmjs.org/@mui/material/-/material-6.4.7.tgz", + "integrity": "sha512-K65StXUeGAtFJ4ikvHKtmDCO5Ab7g0FZUu2J5VpoKD+O6Y3CjLYzRi+TMlI3kaL4CL158+FccMoOd/eaddmeRQ==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.26.0", - "@mui/core-downloads-tracker": "^6.4.6", - "@mui/system": "^6.4.6", + "@mui/core-downloads-tracker": "^6.4.7", + "@mui/system": "^6.4.7", "@mui/types": "^7.2.21", "@mui/utils": "^6.4.6", "@popperjs/core": "^2.11.8", @@ -1061,7 +1061,7 @@ "peerDependencies": { "@emotion/react": "^11.5.0", "@emotion/styled": "^11.3.0", - "@mui/material-pigment-css": "^6.4.6", + "@mui/material-pigment-css": "^6.4.7", "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", "react": "^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" @@ -1143,9 +1143,9 @@ } }, "node_modules/@mui/system": { - "version": "6.4.6", - "resolved": "https://registry.npmjs.org/@mui/system/-/system-6.4.6.tgz", - "integrity": "sha512-FQjWwPec7pMTtB/jw5f9eyLynKFZ6/Ej9vhm5kGdtmts1z5b7Vyn3Rz6kasfYm1j2TfrfGnSXRvvtwVWxjpz6g==", + "version": "6.4.7", + "resolved": "https://registry.npmjs.org/@mui/system/-/system-6.4.7.tgz", + "integrity": "sha512-7wwc4++Ak6tGIooEVA9AY7FhH2p9fvBMORT4vNLMAysH3Yus/9B9RYMbrn3ANgsOyvT3Z7nE+SP8/+3FimQmcg==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.26.0", diff --git a/scripts/settings/package.json b/scripts/settings/package.json index fc7bb7b..5315619 100644 --- a/scripts/settings/package.json +++ b/scripts/settings/package.json @@ -13,7 +13,7 @@ "@dnd-kit/sortable": "^10.0.0", "@emotion/react": "latest", "@emotion/styled": "latest", - "@mui/icons-material": "latest", + "@mui/icons-material": "^6.4.7", "@mui/material": "latest", "react": "19.0.0", "react-dom": "19.0.0", diff --git a/scripts/settings/src/App.tsx b/scripts/settings/src/App.tsx index 4d98528..8e30900 100644 --- a/scripts/settings/src/App.tsx +++ b/scripts/settings/src/App.tsx @@ -4,7 +4,7 @@ import Container from '@mui/material/Container'; import Typography from '@mui/material/Typography'; import Box from '@mui/material/Box'; import FileUploadIcon from '@mui/icons-material/FileUpload'; -// + import { DndContext, closestCenter, @@ -204,7 +204,7 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues {stepType} - Select the {stepType} you wish to enable. Drag to reorder. + Select the {stepType} you wish to enable. Drag to reorder. {showError ? Only one {stepType.slice(0,-1)} can be enabled at a time. : null} diff --git a/scripts/settings/src/schema.json b/scripts/settings/src/schema.json deleted file mode 100644 index 70eb71b..0000000 --- a/scripts/settings/src/schema.json +++ /dev/null @@ -1,2078 +0,0 @@ -{ - "modules": { - "atlos_feeder_db_storage": { - "name": "atlos_feeder_db_storage", - "display_name": "Atlos Feeder Database Storage", - "manifest": { - "name": "Atlos Feeder Database Storage", - "author": "Bellingcat", - "type": [ - "feeder", - "database", - "storage" - ], - "requires_setup": true, - "description": "\n A module that integrates with the Atlos API to fetch source material URLs for archival, uplaod extracted media,\n \n [Atlos](https://www.atlos.org/) is a visual investigation and archiving platform designed for investigative research, journalism, and open-source intelligence (OSINT). \n It helps users organize, analyze, and store media from various sources, making it easier to track and investigate digital evidence.\n \n To get started create a new project and obtain an API token from the settings page. You can group event's into Atlos's 'incidents'.\n Here you can add 'source material' by URLn and the Atlos feeder will fetch these URLs for archival.\n \n You can use Atlos only as a 'feeder', however you can also implement the 'database' and 'storage' features to store the media files in Atlos which is recommended.\n The Auto Archiver will retain the Atlos ID for each item, ensuring that the media and database outputs are uplaoded back into the relevant media item.\n \n \n ### Features\n - Connects to the Atlos API to retrieve a list of source material URLs.\n - Iterates through the URLs from all source material items which are unprocessed, visible, and ready to archive.\n - If the storage option is selected, it will store the media files alongside the original source material item in Atlos.\n - Is the database option is selected it will output the results to the media item, as well as updating failure status with error details when archiving fails.\n - Skips Storege/ database upload for items without an Atlos ID - restricting that you must use the Atlos feeder so that it has the Atlos ID to store the results with.\n\n ### Notes\n - Requires an Atlos account with a project and a valid API token for authentication.\n - Ensures only unprocessed, visible, and ready-to-archive URLs are returned.\n - Feches any media items within an Atlos project, regardless of separation into incidents.\n ", - "dependencies": { - "python": [ - "loguru", - "requests" - ] - }, - "entry_point": "atlos_feeder_db_storage::AtlosFeederDbStorage", - "version": "1.0", - "configs": { - "api_token": { - "type": "str", - "required": true, - "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/" - }, - "atlos_url": { - "default": "https://platform.atlos.org", - "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", - "type": "str" - } - } - }, - "configs": { - "api_token": { - "type": "str", - "required": true, - "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/" - }, - "atlos_url": { - "default": "https://platform.atlos.org", - "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", - "type": "str" - } - } - }, - "csv_feeder": { - "name": "csv_feeder", - "display_name": "CSV Feeder", - "manifest": { - "name": "CSV Feeder", - "author": "Bellingcat", - "type": [ - "feeder" - ], - "requires_setup": true, - "description": "\n Reads URLs from CSV files and feeds them into the archiving process.\n\n ### Features\n - Supports reading URLs from multiple input files, specified as a comma-separated list.\n - Allows specifying the column number or name to extract URLs from.\n - Skips header rows if the first value is not a valid URL.\n\n ### Setup\n - Input files should be formatted with one URL per line, with or without a header row.\n - If you have a header row, you can specify the column number or name to read URLs from using the 'column' config option.\n ", - "dependencies": { - "python": [ - "loguru" - ], - "bin": [ - "" - ] - }, - "entry_point": "csv_feeder::CSVFeeder", - "version": "1.0", - "configs": { - "files": { - "default": null, - "help": "Path to the input file(s) to read the URLs from, comma separated. Input files should be formatted with one URL per line", - "required": true, - "type": "valid_file", - "nargs": "+" - }, - "column": { - "default": null, - "help": "Column number or name to read the URLs from, 0-indexed" - } - } - }, - "configs": { - "files": { - "default": null, - "help": "Path to the input file(s) to read the URLs from, comma separated. Input files should be formatted with one URL per line", - "required": true, - "type": "valid_file", - "nargs": "+" - }, - "column": { - "default": null, - "help": "Column number or name to read the URLs from, 0-indexed" - } - } - }, - "gsheet_feeder_db": { - "name": "gsheet_feeder_db", - "display_name": "Google Sheets Feeder Database", - "manifest": { - "name": "Google Sheets Feeder Database", - "author": "Bellingcat", - "type": [ - "feeder", - "database" - ], - "requires_setup": true, - "description": "\n GsheetsFeederDatabase\n A Google Sheets-based feeder and optional database for the Auto Archiver.\n\n This reads data from Google Sheets and filters rows based on user-defined rules.\n The filtered rows are processed into `Metadata` objects.\n\n ### Features\n - Validates the sheet structure and filters rows based on input configurations.\n - Processes only worksheets allowed by the `allow_worksheets` and `block_worksheets` configurations.\n - Ensures only rows with valid URLs and unprocessed statuses are included for archival.\n - Supports organizing stored files into folder paths based on sheet and worksheet names.\n - If the database is enabled, this updates the Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used.\n - Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns.\n - Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet.\n - Skips redundant updates for empty or invalid data fields.\n\n ### Setup\n - Requires a Google Service Account JSON file for authentication, which should be stored in `secrets/gsheets_service_account.json`.\n To set up a service account, follow the instructions [here](https://gspread.readthedocs.io/en/latest/oauth2.html).\n - Define the `sheet` or `sheet_id` configuration to specify the sheet to archive.\n - Customize the column names in your Google sheet using the `columns` configuration.\n - The Google Sheet can be used soley as a feeder or as a feeder and database, but note you can't currently feed into the database from an alternate feeder.\n ", - "dependencies": { - "python": [ - "loguru", - "gspread", - "slugify" - ] - }, - "entry_point": "gsheet_feeder_db::GsheetsFeederDB", - "version": "1.0", - "configs": { - "sheet": { - "default": null, - "help": "name of the sheet to archive" - }, - "sheet_id": { - "default": null, - "help": "the id of the sheet to archive (alternative to 'sheet' config)" - }, - "header": { - "default": 1, - "type": "int", - "help": "index of the header row (starts at 1)" - }, - "service_account": { - "default": "secrets/service_account.json", - "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html", - "required": true - }, - "columns": { - "default": { - "url": "link", - "status": "archive status", - "folder": "destination folder", - "archive": "archive location", - "date": "archive date", - "thumbnail": "thumbnail", - "timestamp": "upload timestamp", - "title": "upload title", - "text": "text content", - "screenshot": "screenshot", - "hash": "hash", - "pdq_hash": "perceptual hashes", - "wacz": "wacz", - "replaywebpage": "replaywebpage" - }, - "help": "Custom names for the columns in your Google sheet. If you don't want to use the default column names, change them with this setting", - "type": "json_loader" - }, - "allow_worksheets": { - "default": [], - "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed" - }, - "block_worksheets": { - "default": [], - "help": "(CSV) explicitly block some worksheets from being processed" - }, - "use_sheet_names_in_stored_paths": { - "default": true, - "type": "bool", - "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'" - } - } - }, - "configs": { - "sheet": { - "default": null, - "help": "name of the sheet to archive" - }, - "sheet_id": { - "default": null, - "help": "the id of the sheet to archive (alternative to 'sheet' config)" - }, - "header": { - "default": 1, - "type": "int", - "help": "index of the header row (starts at 1)" - }, - "service_account": { - "default": "secrets/service_account.json", - "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html", - "required": true - }, - "columns": { - "default": { - "url": "link", - "status": "archive status", - "folder": "destination folder", - "archive": "archive location", - "date": "archive date", - "thumbnail": "thumbnail", - "timestamp": "upload timestamp", - "title": "upload title", - "text": "text content", - "screenshot": "screenshot", - "hash": "hash", - "pdq_hash": "perceptual hashes", - "wacz": "wacz", - "replaywebpage": "replaywebpage" - }, - "help": "Custom names for the columns in your Google sheet. If you don't want to use the default column names, change them with this setting", - "type": "json_loader" - }, - "allow_worksheets": { - "default": [], - "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed" - }, - "block_worksheets": { - "default": [], - "help": "(CSV) explicitly block some worksheets from being processed" - }, - "use_sheet_names_in_stored_paths": { - "default": true, - "type": "bool", - "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'" - } - } - }, - "cli_feeder": { - "name": "cli_feeder", - "display_name": "Command Line Feeder", - "manifest": { - "name": "Command Line Feeder", - "author": "Bellingcat", - "type": [ - "feeder" - ], - "requires_setup": false, - "description": "\nThe Command Line Feeder is the default enabled feeder for the Auto Archiver. It allows you to pass URLs directly to the orchestrator from the command line \nwithout the need to specify any additional configuration or command line arguments:\n\n`auto-archiver --feeder cli_feeder -- \"https://example.com/1/,https://example.com/2/\"`\n\nYou can pass multiple URLs by separating them with a space. The URLs will be processed in the order they are provided.\n\n`auto-archiver --feeder cli_feeder -- https://example.com/1/ https://example.com/2/`\n", - "dependencies": {}, - "entry_point": "cli_feeder::CLIFeeder", - "version": "1.0", - "configs": { - "urls": { - "default": null, - "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml" - } - } - }, - "configs": { - "urls": { - "default": null, - "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml" - } - } - }, - "instagram_api_extractor": { - "name": "instagram_api_extractor", - "display_name": "Instagram API Extractor", - "manifest": { - "name": "Instagram API Extractor", - "author": "Bellingcat", - "type": [ - "extractor" - ], - "requires_setup": true, - "description": "\nArchives various types of Instagram content using the Instagrapi API.\n\nRequires setting up an Instagrapi API deployment and providing an access token and API endpoint.\n\n### Features\n- Connects to an Instagrapi API deployment to fetch Instagram profiles, posts, stories, highlights, reels, and tagged content.\n- Supports advanced configuration options, including:\n - Full profile download (all posts, stories, highlights, and tagged content).\n - Limiting the number of posts to fetch for large profiles.\n - Minimising JSON output to remove empty fields and redundant data.\n- Provides robust error handling and retries for API calls.\n- Ensures efficient media scraping, including handling nested or carousel media items.\n- Adds downloaded media and metadata to the result for further processing.\n\n### Notes\n- Requires a valid Instagrapi API token (`access_token`) and API endpoint (`api_endpoint`).\n- Full-profile downloads can be limited by setting `full_profile_max_posts`.\n- Designed to fetch content in batches for large profiles, minimising API load.\n", - "dependencies": { - "python": [ - "requests", - "loguru", - "retrying", - "tqdm" - ] - }, - "entry_point": "instagram_api_extractor::InstagramAPIExtractor", - "version": "1.0", - "configs": { - "access_token": { - "default": null, - "help": "a valid instagrapi-api token" - }, - "api_endpoint": { - "required": true, - "help": "API endpoint to use" - }, - "full_profile": { - "default": false, - "type": "bool", - "help": "if true, will download all posts, tagged posts, stories, and highlights for a profile, if false, will only download the profile pic and information." - }, - "full_profile_max_posts": { - "default": 0, - "type": "int", - "help": "Use to limit the number of posts to download when full_profile is true. 0 means no limit. limit is applied softly since posts are fetched in batch, once to: posts, tagged posts, and highlights" - }, - "minimize_json_output": { - "default": true, - "type": "bool", - "help": "if true, will remove empty values from the json output" - } - } - }, - "configs": { - "access_token": { - "default": null, - "help": "a valid instagrapi-api token" - }, - "api_endpoint": { - "required": true, - "help": "API endpoint to use" - }, - "full_profile": { - "default": false, - "type": "bool", - "help": "if true, will download all posts, tagged posts, stories, and highlights for a profile, if false, will only download the profile pic and information." - }, - "full_profile_max_posts": { - "default": 0, - "type": "int", - "help": "Use to limit the number of posts to download when full_profile is true. 0 means no limit. limit is applied softly since posts are fetched in batch, once to: posts, tagged posts, and highlights" - }, - "minimize_json_output": { - "default": true, - "type": "bool", - "help": "if true, will remove empty values from the json output" - } - } - }, - "instagram_tbot_extractor": { - "name": "instagram_tbot_extractor", - "display_name": "Instagram Telegram Bot Extractor", - "manifest": { - "name": "Instagram Telegram Bot Extractor", - "author": "Bellingcat", - "type": [ - "extractor" - ], - "requires_setup": true, - "description": "\nThe `InstagramTbotExtractor` module uses a Telegram bot (`instagram_load_bot`) to fetch and archive Instagram content,\nsuch as posts and stories. It leverages the Telethon library to interact with the Telegram API, sending Instagram URLs\nto the bot and downloading the resulting media and metadata. The downloaded content is stored as `Media` objects and\nreturned as part of a `Metadata` object.\n\n### Features\n- Supports archiving Instagram posts and stories through the Telegram bot.\n- Downloads and saves media files (e.g., images, videos) in a temporary directory.\n- Captures and returns metadata, including titles and descriptions, as a `Metadata` object.\n- Automatically manages Telegram session files for secure access.\n\n### Setup\n\nTo use the `InstagramTbotExtractor`, you need to provide the following configuration settings:\n- **API ID and Hash**: Telegram API credentials obtained from [my.telegram.org/apps](https://my.telegram.org/apps).\n- **Session File**: Optional path to store the Telegram session file for future use.\n- The session file is created automatically and should be unique for each instance.\n- You may need to enter your Telegram credentials (phone) and use the a 2FA code sent to you the first time you run the extractor.:\n```2025-01-30 00:43:49.348 | INFO | auto_archiver.modules.instagram_tbot_extractor.instagram_tbot_extractor:setup:36 - SETUP instagram_tbot_extractor checking login...\nPlease enter your phone (or bot token): +447123456789\nPlease enter the code you received: 00000\nSigned in successfully as E C; remember to not break the ToS or you will risk an account ban!\n```\n ", - "dependencies": { - "python": [ - "loguru", - "telethon" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "api_id": { - "default": null, - "help": "telegram API_ID value, go to https://my.telegram.org/apps" - }, - "api_hash": { - "default": null, - "help": "telegram API_HASH value, go to https://my.telegram.org/apps" - }, - "session_file": { - "default": "secrets/anon-insta", - "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value." - }, - "timeout": { - "default": 45, - "type": "int", - "help": "timeout to fetch the instagram content in seconds." - } - } - }, - "configs": { - "api_id": { - "default": null, - "help": "telegram API_ID value, go to https://my.telegram.org/apps" - }, - "api_hash": { - "default": null, - "help": "telegram API_HASH value, go to https://my.telegram.org/apps" - }, - "session_file": { - "default": "secrets/anon-insta", - "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value." - }, - "timeout": { - "default": 45, - "type": "int", - "help": "timeout to fetch the instagram content in seconds." - } - } - }, - "twitter_api_extractor": { - "name": "twitter_api_extractor", - "display_name": "Twitter API Extractor", - "manifest": { - "name": "Twitter API Extractor", - "author": "Bellingcat", - "type": [ - "extractor" - ], - "requires_setup": true, - "description": "\n The `TwitterApiExtractor` fetches tweets and associated media using the Twitter API. \n It supports multiple API configurations for extended rate limits and reliable access. \n Features include URL expansion, media downloads (e.g., images, videos), and structured output \n via `Metadata` and `Media` objects. Requires Twitter API credentials such as bearer tokens \n or consumer key/secret and access token/secret.\n \n ### Features\n - Fetches tweets and their metadata, including text, creation timestamp, and author information.\n - Downloads media attachments (e.g., images, videos) in high quality.\n - Supports multiple API configurations for improved rate limiting.\n - Expands shortened URLs (e.g., `t.co` links).\n - Outputs structured metadata and media using `Metadata` and `Media` objects.\n \n ### Setup\n To use the `TwitterApiExtractor`, you must provide valid Twitter API credentials via configuration:\n - **Bearer Token(s)**: A single token or a list for rate-limited API access.\n - **Consumer Key and Secret**: Required for user-authenticated API access.\n - **Access Token and Secret**: Complements the consumer key for enhanced API capabilities.\n \n Credentials can be obtained by creating a Twitter developer account at [Twitter Developer Platform](https://developer.twitter.com/en).\n ", - "dependencies": { - "python": [ - "requests", - "loguru", - "pytwitter", - "slugify" - ], - "bin": [ - "" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "bearer_token": { - "default": null, - "help": "[deprecated: see bearer_tokens] twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret" - }, - "bearer_tokens": { - "default": [], - "help": " a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line" - }, - "consumer_key": { - "default": null, - "help": "twitter API consumer_key" - }, - "consumer_secret": { - "default": null, - "help": "twitter API consumer_secret" - }, - "access_token": { - "default": null, - "help": "twitter API access_token" - }, - "access_secret": { - "default": null, - "help": "twitter API access_secret" - } - } - }, - "configs": { - "bearer_token": { - "default": null, - "help": "[deprecated: see bearer_tokens] twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret" - }, - "bearer_tokens": { - "default": [], - "help": " a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line" - }, - "consumer_key": { - "default": null, - "help": "twitter API consumer_key" - }, - "consumer_secret": { - "default": null, - "help": "twitter API consumer_secret" - }, - "access_token": { - "default": null, - "help": "twitter API access_token" - }, - "access_secret": { - "default": null, - "help": "twitter API access_secret" - } - } - }, - "instagram_extractor": { - "name": "instagram_extractor", - "display_name": "Instagram Extractor", - "manifest": { - "name": "Instagram Extractor", - "author": "Bellingcat", - "type": [ - "extractor" - ], - "requires_setup": true, - "description": "\n Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. \n \n > \u26a0\ufe0f **Warning** \n > This module is not actively maintained due to known issues with blocking. \n > Prioritise usage of the [Instagram Tbot Extractor](./instagram_tbot_extractor.md) and [Instagram API Extractor](./instagram_api_extractor.md)\n \n This class handles both individual posts and user profiles, downloading as much information as possible, including images, videos, text, stories,\n highlights, and tagged posts. \n Authentication is required via username/password or a session file.\n \n ", - "dependencies": { - "python": [ - "instaloader", - "loguru" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "username": { - "required": true, - "help": "A valid Instagram username." - }, - "password": { - "required": true, - "help": "The corresponding Instagram account password." - }, - "download_folder": { - "default": "instaloader", - "help": "Name of a folder to temporarily download content to." - }, - "session_file": { - "default": "secrets/instaloader.session", - "help": "Path to the instagram session file which saves session credentials. If one doesn't exist this gives the path to store a new one." - } - } - }, - "configs": { - "username": { - "required": true, - "help": "A valid Instagram username." - }, - "password": { - "required": true, - "help": "The corresponding Instagram account password." - }, - "download_folder": { - "default": "instaloader", - "help": "Name of a folder to temporarily download content to." - }, - "session_file": { - "default": "secrets/instaloader.session", - "help": "Path to the instagram session file which saves session credentials. If one doesn't exist this gives the path to store a new one." - } - } - }, - "telethon_extractor": { - "name": "telethon_extractor", - "display_name": "Telethon Extractor", - "manifest": { - "name": "Telethon Extractor", - "author": "Bellingcat", - "type": [ - "extractor" - ], - "requires_setup": true, - "description": "\nThe `TelethonExtractor` uses the Telethon library to archive posts and media from Telegram channels and groups. \nIt supports private and public channels, downloading grouped posts with media, and can join channels using invite links \nif provided in the configuration. \n\n### Features\n- Fetches posts and metadata from Telegram channels and groups, including private channels.\n- Downloads media attachments (e.g., images, videos, audio) from individual posts or grouped posts.\n- Handles channel invites to join channels dynamically during setup.\n- Utilizes Telethon's capabilities for reliable Telegram interactions.\n- Outputs structured metadata and media using `Metadata` and `Media` objects.\n\n### Setup\nTo use the `TelethonExtractor`, you must configure the following:\n- **API ID and API Hash**: Obtain these from [my.telegram.org](https://my.telegram.org/apps).\n- **Session File**: Optional, but records login sessions for future use (default: `secrets/anon.session`).\n- **Bot Token**: Optional, allows access to additional content (e.g., large videos) but limits private channel archiving.\n- **Channel Invites**: Optional, specify a JSON string of invite links to join channels during setup.\n\n### First Time Login\nThe first time you run, you will be prompted to do a authentication with the phone number associated, alternatively you can put your `anon.session` in the root.\n\n\n", - "dependencies": { - "python": [ - "telethon", - "loguru", - "tqdm" - ], - "bin": [ - "" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "api_id": { - "default": null, - "help": "telegram API_ID value, go to https://my.telegram.org/apps" - }, - "api_hash": { - "default": null, - "help": "telegram API_HASH value, go to https://my.telegram.org/apps" - }, - "bot_token": { - "default": null, - "help": "optional, but allows access to more content such as large videos, talk to @botfather" - }, - "session_file": { - "default": "secrets/anon", - "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value." - }, - "join_channels": { - "default": true, - "type": "bool", - "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck" - }, - "channel_invites": { - "default": {}, - "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup", - "type": "json_loader" - } - } - }, - "configs": { - "api_id": { - "default": null, - "help": "telegram API_ID value, go to https://my.telegram.org/apps" - }, - "api_hash": { - "default": null, - "help": "telegram API_HASH value, go to https://my.telegram.org/apps" - }, - "bot_token": { - "default": null, - "help": "optional, but allows access to more content such as large videos, talk to @botfather" - }, - "session_file": { - "default": "secrets/anon", - "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value." - }, - "join_channels": { - "default": true, - "type": "bool", - "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck" - }, - "channel_invites": { - "default": {}, - "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup", - "type": "json_loader" - } - } - }, - "vk_extractor": { - "name": "vk_extractor", - "display_name": "VKontakte Extractor", - "manifest": { - "name": "VKontakte Extractor", - "author": "Bellingcat", - "type": [ - "extractor" - ], - "requires_setup": true, - "description": "\nThe `VkExtractor` fetches posts, text, and images from VK (VKontakte) social media pages. \nThis archiver is specialized for `/wall` posts and uses the `VkScraper` library to extract \nand download content. Note that VK videos are handled separately by the `YTDownloader`.\n\n### Features\n- Extracts text, timestamps, and metadata from VK `/wall` posts.\n- Downloads associated images and attaches them to the resulting `Metadata` object.\n- Processes multiple segments of VK URLs that contain mixed content (e.g., wall, photo).\n- Outputs structured metadata and media using `Metadata` and `Media` objects.\n\n### Setup\nTo use the `VkArchiver`, you must provide valid VKontakte login credentials and session information:\n- **Username**: A valid VKontakte account username.\n- **Password**: The corresponding password for the VKontakte account.\n- **Session File**: Optional. Path to a session configuration file (`.json`) for persistent VK login.\n\nCredentials can be set in the configuration file or directly via environment variables. Ensure you \nhave access to the VKontakte API by creating an account at [VKontakte](https://vk.com/).\n", - "dependencies": { - "python": [ - "loguru", - "vk_url_scraper" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "username": { - "required": true, - "help": "valid VKontakte username" - }, - "password": { - "required": true, - "help": "valid VKontakte password" - }, - "session_file": { - "default": "secrets/vk_config.v2.json", - "help": "valid VKontakte password" - } - }, - "depends": [ - "core", - "utils" - ] - }, - "configs": { - "username": { - "required": true, - "help": "valid VKontakte username" - }, - "password": { - "required": true, - "help": "valid VKontakte password" - }, - "session_file": { - "default": "secrets/vk_config.v2.json", - "help": "valid VKontakte password" - } - } - }, - "generic_extractor": { - "name": "generic_extractor", - "display_name": "Generic Extractor", - "manifest": { - "name": "Generic Extractor", - "author": "Bellingcat", - "type": [ - "extractor" - ], - "requires_setup": false, - "description": "\nThis is the generic extractor used by auto-archiver, which uses `yt-dlp` under the hood.\n\nThis module is responsible for downloading and processing media content from platforms\nsupported by `yt-dlp`, such as YouTube, Facebook, and others. It provides functionality\nfor retrieving videos, subtitles, comments, and other metadata, and it integrates with\nthe broader archiving framework.\n\n### Features\n- Supports downloading videos and playlists.\n- Retrieves metadata like titles, descriptions, upload dates, and durations.\n- Downloads subtitles and comments when enabled.\n- Configurable options for handling live streams, proxies, and more.\n- Supports authentication of websites using the 'authentication' settings from your orchestration.\n\n### Dropins\n- For websites supported by `yt-dlp` that also contain posts in addition to videos\n (e.g. Facebook, Twitter, Bluesky), dropins can be created to extract post data and create \n metadata objects. Some dropins are included in this generic_archiver by default, but\ncustom dropins can be created to handle additional websites and passed to the archiver\nvia the command line using the `--dropins` option (TODO!).\n\n### Auto-Updates\n\nThe Generic Extractor will also automatically check for updates to `yt-dlp` (every 5 days by default).\nThis can be configured using the `ytdlp_update_interval` setting (or disabled by setting it to -1).\nIf you are having issues with the extractor, you can review the version of `yt-dlp` being used with `yt-dlp --version`.\n\n", - "dependencies": { - "python": [ - "yt_dlp", - "requests", - "loguru", - "slugify" - ] - }, - "entry_point": "", - "version": "0.1.0", - "configs": { - "subtitles": { - "default": true, - "help": "download subtitles if available", - "type": "bool" - }, - "comments": { - "default": false, - "help": "download all comments if available, may lead to large metadata", - "type": "bool" - }, - "livestreams": { - "default": false, - "help": "if set, will download live streams, otherwise will skip them; see --max-filesize for more control", - "type": "bool" - }, - "live_from_start": { - "default": false, - "help": "if set, will download live streams from their earliest available moment, otherwise starts now.", - "type": "bool" - }, - "proxy": { - "default": "", - "help": "http/socks (https seems to not work atm) proxy to use for the webdriver, eg https://proxy-user:password@proxy-ip:port" - }, - "end_means_success": { - "default": true, - "help": "if True, any archived content will mean a 'success', if False this archiver will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent archivers can retrieve.", - "type": "bool" - }, - "allow_playlist": { - "default": false, - "help": "If True will also download playlists, set to False if the expectation is to download a single video.", - "type": "bool" - }, - "max_downloads": { - "default": "inf", - "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit." - }, - "ytdlp_update_interval": { - "default": 5, - "help": "How often to check for yt-dlp updates (days). If positive, will check and update yt-dlp every [num] days. Set it to -1 to disable, or 0 to always update on every run.", - "type": "int" - } - } - }, - "configs": { - "subtitles": { - "default": true, - "help": "download subtitles if available", - "type": "bool" - }, - "comments": { - "default": false, - "help": "download all comments if available, may lead to large metadata", - "type": "bool" - }, - "livestreams": { - "default": false, - "help": "if set, will download live streams, otherwise will skip them; see --max-filesize for more control", - "type": "bool" - }, - "live_from_start": { - "default": false, - "help": "if set, will download live streams from their earliest available moment, otherwise starts now.", - "type": "bool" - }, - "proxy": { - "default": "", - "help": "http/socks (https seems to not work atm) proxy to use for the webdriver, eg https://proxy-user:password@proxy-ip:port" - }, - "end_means_success": { - "default": true, - "help": "if True, any archived content will mean a 'success', if False this archiver will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent archivers can retrieve.", - "type": "bool" - }, - "allow_playlist": { - "default": false, - "help": "If True will also download playlists, set to False if the expectation is to download a single video.", - "type": "bool" - }, - "max_downloads": { - "default": "inf", - "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit." - }, - "ytdlp_update_interval": { - "default": 5, - "help": "How often to check for yt-dlp updates (days). If positive, will check and update yt-dlp every [num] days. Set it to -1 to disable, or 0 to always update on every run.", - "type": "int" - } - } - }, - "tiktok_tikwm_extractor": { - "name": "tiktok_tikwm_extractor", - "display_name": "Tiktok Tikwm Extractor", - "manifest": { - "name": "Tiktok Tikwm Extractor", - "author": "Bellingcat", - "type": [ - "extractor" - ], - "requires_setup": false, - "description": "\n Uses an unofficial TikTok video download platform's API to download videos: https://tikwm.com/\n\t\n\tThis extractor complements the generic_extractor which can already get TikTok videos, but this one can extract special videos like those marked as sensitive.\n\n ### Features\n - Downloads the video and, if possible, also the video cover.\n\t- Stores extra metadata about the post like author information, and more as returned by tikwm.com. \n\n ### Notes\n - If tikwm.com is down, this extractor will not work.\n\t- If tikwm.com changes their API, this extractor may break.\n\t- If no video is found, this extractor will consider the extraction failed.\n ", - "dependencies": { - "python": [ - "loguru", - "requests" - ], - "bin": [] - }, - "entry_point": "", - "version": "1.0", - "configs": {} - }, - "configs": null - }, - "telegram_extractor": { - "name": "telegram_extractor", - "display_name": "Telegram Extractor", - "manifest": { - "name": "Telegram Extractor", - "author": "Bellingcat", - "type": [ - "extractor" - ], - "requires_setup": false, - "description": " \n The `TelegramExtractor` retrieves publicly available media content from Telegram message links without requiring login credentials. \n It processes URLs to fetch images and videos embedded in Telegram messages, ensuring a structured output using `Metadata` \n and `Media` objects. Recommended for scenarios where login-based archiving is not viable, although `telethon_archiver` \n is advised for more comprehensive functionality, and higher quality media extraction.\n \n ### Features\n- Extracts images and videos from public Telegram message links (`t.me`).\n- Processes HTML content of messages to retrieve embedded media.\n- Sets structured metadata, including timestamps, content, and media details.\n- Does not require user authentication for Telegram.\n\n ", - "dependencies": { - "python": [ - "requests", - "bs4", - "loguru" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": {} - }, - "configs": null - }, - "wayback_extractor_enricher": { - "name": "wayback_extractor_enricher", - "display_name": "Wayback Machine Enricher (and Extractor)", - "manifest": { - "name": "Wayback Machine Enricher (and Extractor)", - "author": "Bellingcat", - "type": [ - "enricher", - "extractor" - ], - "requires_setup": true, - "description": "\n Submits the current URL to the Wayback Machine for archiving and returns either a job ID or the completed archive URL.\n\n ### Features\n - Archives URLs using the Internet Archive's Wayback Machine API.\n - Supports conditional archiving based on the existence of prior archives within a specified time range.\n - Provides proxies for HTTP and HTTPS requests.\n - Fetches and confirms the archive URL or provides a job ID for later status checks.\n\n ### Notes\n - Requires a valid Wayback Machine API key and secret.\n - Handles rate-limiting by Wayback Machine and retries status checks with exponential backoff.\n \n ### Steps to Get an Wayback API Key:\n - Sign up for an account at [Internet Archive](https://archive.org/account/signup).\n - Log in to your account.\n - Navigte to your [account settings](https://archive.org/account).\n - or: https://archive.org/developers/tutorial-get-ia-credentials.html\n - Under Wayback Machine API Keys, generate a new key.\n - Note down your API key and secret, as they will be required for authentication.\n ", - "dependencies": { - "python": [ - "loguru", - "requests" - ] - }, - "entry_point": "wayback_extractor_enricher::WaybackExtractorEnricher", - "version": "1.0", - "configs": { - "timeout": { - "default": 15, - "type": "int", - "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually." - }, - "if_not_archived_within": { - "default": null, - "help": "only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA" - }, - "key": { - "required": true, - "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php" - }, - "secret": { - "required": true, - "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php" - }, - "proxy_http": { - "default": null, - "help": "http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port" - }, - "proxy_https": { - "default": null, - "help": "https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port" - } - } - }, - "configs": { - "timeout": { - "default": 15, - "type": "int", - "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually." - }, - "if_not_archived_within": { - "default": null, - "help": "only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA" - }, - "key": { - "required": true, - "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php" - }, - "secret": { - "required": true, - "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php" - }, - "proxy_http": { - "default": null, - "help": "http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port" - }, - "proxy_https": { - "default": null, - "help": "https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port" - } - } - }, - "wacz_extractor_enricher": { - "name": "wacz_extractor_enricher", - "display_name": "WACZ Enricher (and Extractor)", - "manifest": { - "name": "WACZ Enricher (and Extractor)", - "author": "Bellingcat", - "type": [ - "enricher", - "extractor" - ], - "requires_setup": true, - "description": "\n Creates .WACZ archives of web pages using the `browsertrix-crawler` tool, with options for media extraction and screenshot saving.\n [Browsertrix-crawler](https://crawler.docs.browsertrix.com/user-guide/) is a headless browser-based crawler that archives web pages in WACZ format.\n\n ### Features\n - Archives web pages into .WACZ format using Docker or direct invocation of `browsertrix-crawler`.\n - Supports custom profiles for archiving private or dynamic content.\n - Extracts media (images, videos, audio) and screenshots from the archive, optionally adding them to the enrichment pipeline.\n - Generates metadata from the archived page's content and structure (e.g., titles, text).\n\n ### Notes\n - Requires Docker for running `browsertrix-crawler` .\n - Configurable via parameters for timeout, media extraction, screenshots, and proxy settings.\n ", - "dependencies": { - "python": [ - "loguru", - "jsonlines", - "warcio" - ], - "bin": [ - "docker" - ] - }, - "entry_point": "wacz_extractor_enricher::WaczExtractorEnricher", - "version": "1.0", - "configs": { - "profile": { - "default": null, - "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)." - }, - "docker_commands": { - "default": null, - "help": "if a custom docker invocation is needed" - }, - "timeout": { - "default": 120, - "type": "int", - "help": "timeout for WACZ generation in seconds" - }, - "extract_media": { - "default": false, - "type": "bool", - "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." - }, - "extract_screenshot": { - "default": true, - "type": "bool", - "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." - }, - "socks_proxy_host": { - "default": null, - "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host" - }, - "socks_proxy_port": { - "default": null, - "type": "int", - "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234" - }, - "proxy_server": { - "default": null, - "help": "SOCKS server proxy URL, in development" - } - } - }, - "configs": { - "profile": { - "default": null, - "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)." - }, - "docker_commands": { - "default": null, - "help": "if a custom docker invocation is needed" - }, - "timeout": { - "default": 120, - "type": "int", - "help": "timeout for WACZ generation in seconds" - }, - "extract_media": { - "default": false, - "type": "bool", - "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." - }, - "extract_screenshot": { - "default": true, - "type": "bool", - "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." - }, - "socks_proxy_host": { - "default": null, - "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host" - }, - "socks_proxy_port": { - "default": null, - "type": "int", - "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234" - }, - "proxy_server": { - "default": null, - "help": "SOCKS server proxy URL, in development" - } - } - }, - "metadata_enricher": { - "name": "metadata_enricher", - "display_name": "Media Metadata Enricher", - "manifest": { - "name": "Media Metadata Enricher", - "author": "Bellingcat", - "type": [ - "enricher" - ], - "requires_setup": true, - "description": "\n Extracts metadata information from files using ExifTool.\n\n ### Features\n - Uses ExifTool to extract detailed metadata from media files.\n - Processes file-specific data like camera settings, geolocation, timestamps, and other embedded metadata.\n - Adds extracted metadata to the corresponding `Media` object within the `Metadata`.\n\n ### Notes\n - Requires ExifTool to be installed and accessible via the system's PATH.\n - Skips enrichment for files where metadata extraction fails.\n ", - "dependencies": { - "python": [ - "loguru" - ], - "bin": [ - "exiftool" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": {} - }, - "configs": null - }, - "timestamping_enricher": { - "name": "timestamping_enricher", - "display_name": "Timestamping Enricher", - "manifest": { - "name": "Timestamping Enricher", - "author": "Bellingcat", - "type": [ - "enricher" - ], - "requires_setup": true, - "description": "\n Generates RFC3161-compliant timestamp tokens using Time Stamp Authorities (TSA) for archived files.\n\n ### Features\n - Creates timestamp tokens to prove the existence of files at a specific time, useful for legal and authenticity purposes.\n - Aggregates file hashes into a text file and timestamps the concatenated data.\n - Uses multiple Time Stamp Authorities (TSAs) to ensure reliability and redundancy.\n - Validates timestamping certificates against trusted Certificate Authorities (CAs) using the `certifi` trust store.\n\n ### Notes\n - Should be run after the `hash_enricher` to ensure file hashes are available.\n - Requires internet access to interact with the configured TSAs.\n ", - "dependencies": { - "python": [ - "loguru", - "slugify", - "tsp_client", - "asn1crypto", - "certvalidator", - "certifi" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "tsa_urls": { - "default": [ - "http://timestamp.digicert.com", - "http://timestamp.identrust.com", - "http://timestamp.globalsign.com/tsa/r6advanced1", - "http://tss.accv.es:8318/tsa" - ], - "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line." - } - } - }, - "configs": { - "tsa_urls": { - "default": [ - "http://timestamp.digicert.com", - "http://timestamp.identrust.com", - "http://timestamp.globalsign.com/tsa/r6advanced1", - "http://tss.accv.es:8318/tsa" - ], - "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line." - } - } - }, - "screenshot_enricher": { - "name": "screenshot_enricher", - "display_name": "Screenshot Enricher", - "manifest": { - "name": "Screenshot Enricher", - "author": "Bellingcat", - "type": [ - "enricher" - ], - "requires_setup": true, - "description": "\n Captures screenshots and optionally saves web pages as PDFs using a WebDriver.\n\n ### Features\n - Takes screenshots of web pages, with configurable width, height, and timeout settings.\n - Optionally saves pages as PDFs, with additional configuration for PDF printing options.\n - Bypasses URLs detected as authentication walls.\n - Integrates seamlessly with the metadata enrichment pipeline, adding screenshots and PDFs as media.\n\n ### Notes\n - Requires a WebDriver (e.g., ChromeDriver) installed and accessible via the system's PATH.\n ", - "dependencies": { - "python": [ - "loguru", - "selenium" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "width": { - "default": 1280, - "type": "int", - "help": "width of the screenshots" - }, - "height": { - "default": 1024, - "type": "int", - "help": "height of the screenshots" - }, - "timeout": { - "default": 60, - "type": "int", - "help": "timeout for taking the screenshot" - }, - "sleep_before_screenshot": { - "default": 4, - "type": "int", - "help": "seconds to wait for the pages to load before taking screenshot" - }, - "http_proxy": { - "default": "", - "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port" - }, - "save_to_pdf": { - "default": false, - "type": "bool", - "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter" - }, - "print_options": { - "default": {}, - "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information", - "type": "json_loader" - } - } - }, - "configs": { - "width": { - "default": 1280, - "type": "int", - "help": "width of the screenshots" - }, - "height": { - "default": 1024, - "type": "int", - "help": "height of the screenshots" - }, - "timeout": { - "default": 60, - "type": "int", - "help": "timeout for taking the screenshot" - }, - "sleep_before_screenshot": { - "default": 4, - "type": "int", - "help": "seconds to wait for the pages to load before taking screenshot" - }, - "http_proxy": { - "default": "", - "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port" - }, - "save_to_pdf": { - "default": false, - "type": "bool", - "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter" - }, - "print_options": { - "default": {}, - "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information", - "type": "json_loader" - } - } - }, - "whisper_enricher": { - "name": "whisper_enricher", - "display_name": "Whisper Enricher", - "manifest": { - "name": "Whisper Enricher", - "author": "Bellingcat", - "type": [ - "enricher" - ], - "requires_setup": true, - "description": "\n Integrates with a Whisper API service to transcribe, translate, or detect the language of audio and video files.\n\n ### Features\n - Submits audio or video files to a Whisper API deployment for processing.\n - Supports operations such as transcription, translation, and language detection.\n - Optionally generates SRT subtitle files for video content.\n - Integrates with S3-compatible storage systems to make files publicly accessible for processing.\n - Handles job submission, status checking, artifact retrieval, and cleanup.\n\n ### Notes\n - Requires a Whisper API endpoint and API key for authentication.\n - Only compatible with S3-compatible storage systems for media file accessibility.\n - ** This stores the media files in S3 prior to enriching them as Whisper requires public URLs to access the media files.\n - Handles multiple jobs and retries for failed or incomplete processing.\n ", - "dependencies": { - "python": [ - "s3_storage", - "loguru", - "requests" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "api_endpoint": { - "required": true, - "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe." - }, - "api_key": { - "required": true, - "help": "WhisperApi api key for authentication" - }, - "include_srt": { - "default": false, - "type": "bool", - "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)." - }, - "timeout": { - "default": 90, - "type": "int", - "help": "How many seconds to wait at most for a successful job completion." - }, - "action": { - "default": "translate", - "help": "which Whisper operation to execute", - "choices": [ - "transcribe", - "translate", - "language_detection" - ] - } - } - }, - "configs": { - "api_endpoint": { - "required": true, - "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe." - }, - "api_key": { - "required": true, - "help": "WhisperApi api key for authentication" - }, - "include_srt": { - "default": false, - "type": "bool", - "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)." - }, - "timeout": { - "default": 90, - "type": "int", - "help": "How many seconds to wait at most for a successful job completion." - }, - "action": { - "default": "translate", - "help": "which Whisper operation to execute", - "choices": [ - "transcribe", - "translate", - "language_detection" - ] - } - } - }, - "opentimestamps_enricher": { - "name": "opentimestamps_enricher", - "display_name": "OpenTimestamps Enricher", - "manifest": { - "name": "OpenTimestamps Enricher", - "author": "Bellingcat", - "type": [ - "enricher" - ], - "requires_setup": true, - "description": "\n Creates OpenTimestamps proofs for archived files, providing blockchain-backed evidence of file existence at a specific time.\n\n Uses OpenTimestamps \u2013 a service that timestamps data using the Bitcoin blockchain, providing a decentralized \n and secure way to prove that data existed at a certain point in time.\n\n ### Features\n - Creates cryptographic timestamp proofs that link files to the Bitcoin blockchain\n - Verifies existing timestamp proofs to confirm the time a file existed\n - Uses multiple calendar servers to ensure reliability and redundancy\n - Stores timestamp proofs alongside original files for future verification\n\n ### Notes\n - Can work offline to create timestamp proofs that can be upgraded later\n - Verification checks if timestamps have been confirmed in the Bitcoin blockchain\n - Should run after files have been archived and hashed\n\n ### Verifying Timestamps Later\n If you wish to verify a timestamp (ots) file later, you can install the opentimestamps-client command line tool and use the `ots verify` command.\n Example: `ots verify my_file.ots`\n\n Note: if you're using local storage with a filename_generator set to 'static' (a hash) or random, the files will be renamed when they are saved to the\n final location meaning you will need to specify the original filename when verifying the timestamp with `ots verify -f original_filename my_file.ots`.\n ", - "dependencies": { - "python": [ - "loguru", - "opentimestamps" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "use_calendars": { - "default": true, - "help": "Whether to connect to OpenTimestamps calendar servers to create timestamps. If false, creates local timestamp proofs only.", - "type": "bool" - }, - "calendar_urls": { - "default": [ - "https://alice.btc.calendar.opentimestamps.org", - "https://bob.btc.calendar.opentimestamps.org", - "https://finney.calendar.eternitywall.com" - ], - "help": "List of OpenTimestamps calendar servers to use for timestamping. See here for a list of calendars maintained by opentimestamps:https://opentimestamps.org/#calendars", - "type": "list" - }, - "calendar_whitelist": { - "default": [], - "help": "Optional whitelist of calendar servers. Override this if you are using your own calendar servers. e.g. ['https://mycalendar.com']", - "type": "list" - }, - "verify_timestamps": { - "default": true, - "help": "Whether to verify timestamps after creating them.", - "type": "bool" - } - } - }, - "configs": { - "use_calendars": { - "default": true, - "help": "Whether to connect to OpenTimestamps calendar servers to create timestamps. If false, creates local timestamp proofs only.", - "type": "bool" - }, - "calendar_urls": { - "default": [ - "https://alice.btc.calendar.opentimestamps.org", - "https://bob.btc.calendar.opentimestamps.org", - "https://finney.calendar.eternitywall.com" - ], - "help": "List of OpenTimestamps calendar servers to use for timestamping. See here for a list of calendars maintained by opentimestamps:https://opentimestamps.org/#calendars", - "type": "list" - }, - "calendar_whitelist": { - "default": [], - "help": "Optional whitelist of calendar servers. Override this if you are using your own calendar servers. e.g. ['https://mycalendar.com']", - "type": "list" - }, - "verify_timestamps": { - "default": true, - "help": "Whether to verify timestamps after creating them.", - "type": "bool" - } - } - }, - "thumbnail_enricher": { - "name": "thumbnail_enricher", - "display_name": "Thumbnail Enricher", - "manifest": { - "name": "Thumbnail Enricher", - "author": "Bellingcat", - "type": [ - "enricher" - ], - "requires_setup": false, - "description": "\n Generates thumbnails for video files to provide visual previews.\n\n ### Features\n - Processes video files and generates evenly distributed thumbnails.\n - Calculates the number of thumbnails based on video duration, `thumbnails_per_minute`, and `max_thumbnails`.\n - Distributes thumbnails equally across the video's duration and stores them as media objects.\n - Adds metadata for each thumbnail, including timestamps and IDs.\n\n ### Notes\n - Requires `ffmpeg` to be installed and accessible via the system's PATH.\n - Handles videos without pre-existing duration metadata by probing with `ffmpeg`.\n - Skips enrichment for non-video media files.\n ", - "dependencies": { - "python": [ - "loguru", - "ffmpeg" - ], - "bin": [ - "ffmpeg" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "thumbnails_per_minute": { - "default": 60, - "type": "int", - "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails" - }, - "max_thumbnails": { - "default": 16, - "type": "int", - "help": "limit the number of thumbnails to generate per video, 0 means no limit" - } - } - }, - "configs": { - "thumbnails_per_minute": { - "default": 60, - "type": "int", - "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails" - }, - "max_thumbnails": { - "default": 16, - "type": "int", - "help": "limit the number of thumbnails to generate per video, 0 means no limit" - } - } - }, - "meta_enricher": { - "name": "meta_enricher", - "display_name": "Archive Metadata Enricher", - "manifest": { - "name": "Archive Metadata Enricher", - "author": "Bellingcat", - "type": [ - "enricher" - ], - "requires_setup": false, - "description": " \n Adds metadata information about the archive operations, Adds metadata about archive operations, including file sizes and archive duration./\n To be included at the end of all enrichments.\n \n ### Features\n- Calculates the total size of all archived media files, storing the result in human-readable and byte formats.\n- Computes the duration of the archival process, storing the elapsed time in seconds.\n- Ensures all enrichments are performed only if the `Metadata` object contains valid data.\n- Adds detailed metadata to provide insights into file sizes and archival performance.\n\n### Notes\n- Skips enrichment if no media or metadata is available in the `Metadata` object.\n- File sizes are calculated using the `os.stat` module, ensuring accurate byte-level reporting.\n", - "dependencies": { - "python": [ - "loguru" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": {} - }, - "configs": null - }, - "pdq_hash_enricher": { - "name": "pdq_hash_enricher", - "display_name": "PDQ Hash Enricher", - "manifest": { - "name": "PDQ Hash Enricher", - "author": "Bellingcat", - "type": [ - "enricher" - ], - "requires_setup": false, - "description": "\n PDQ Hash Enricher for generating perceptual hashes of media files.\n\n ### Features\n - Calculates perceptual hashes for image files using the PDQ hashing algorithm.\n - Enables detection of duplicate or near-duplicate visual content.\n - Processes images stored in `Metadata` objects, adding computed hashes to the corresponding `Media` entries.\n - Skips non-image media or files unsuitable for hashing (e.g., corrupted or unsupported formats).\n\n ### Notes\n - Best used after enrichers like `thumbnail_enricher` or `screenshot_enricher` to ensure images are available.\n - Uses the `pdqhash` library to compute 256-bit perceptual hashes, which are stored as hexadecimal strings.\n ", - "dependencies": { - "python": [ - "loguru", - "pdqhash", - "numpy", - "PIL" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": {} - }, - "configs": null - }, - "ssl_enricher": { - "name": "ssl_enricher", - "display_name": "SSL Certificate Enricher", - "manifest": { - "name": "SSL Certificate Enricher", - "author": "Bellingcat", - "type": [ - "enricher" - ], - "requires_setup": false, - "description": "\n Retrieves SSL certificate information for a domain and stores it as a file.\n\n ### Features\n - Fetches SSL certificates for domains using the HTTPS protocol.\n - Stores certificates in PEM format and adds them as media to the metadata.\n - Skips enrichment if no media has been archived, based on the `skip_when_nothing_archived` configuration.\n\n ### Notes\n - Requires the target URL to use the HTTPS scheme; other schemes are not supported.\n ", - "dependencies": { - "python": [ - "loguru", - "slugify" - ] - }, - "entry_point": "ssl_enricher::SSLEnricher", - "version": "1.0", - "configs": { - "skip_when_nothing_archived": { - "default": true, - "type": "bool", - "help": "if true, will skip enriching when no media is archived" - } - } - }, - "configs": { - "skip_when_nothing_archived": { - "default": true, - "type": "bool", - "help": "if true, will skip enriching when no media is archived" - } - } - }, - "hash_enricher": { - "name": "hash_enricher", - "display_name": "Hash Enricher", - "manifest": { - "name": "Hash Enricher", - "author": "Bellingcat", - "type": [ - "enricher" - ], - "requires_setup": false, - "description": "\nGenerates cryptographic hashes for media files to ensure data integrity and authenticity.\n\n### Features\n- Calculates cryptographic hashes (SHA-256 or SHA3-512) for media files stored in `Metadata` objects.\n- Ensures content authenticity, integrity validation, and duplicate identification.\n- Efficiently processes large files by reading file bytes in configurable chunk sizes.\n- Supports dynamic configuration of hash algorithms and chunk sizes.\n- Updates media metadata with the computed hash value in the format `:`.\n\n### Notes\n- Default hash algorithm is SHA-256, but SHA3-512 is also supported.\n- Chunk size defaults to 16 MB but can be adjusted based on memory requirements.\n- Useful for workflows requiring hash-based content validation or deduplication.\n", - "dependencies": { - "python": [ - "loguru" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "algorithm": { - "default": "SHA-256", - "help": "hash algorithm to use", - "choices": [ - "SHA-256", - "SHA3-512" - ] - }, - "chunksize": { - "default": 16000000, - "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB", - "type": "int" - } - } - }, - "configs": { - "algorithm": { - "default": "SHA-256", - "help": "hash algorithm to use", - "choices": [ - "SHA-256", - "SHA3-512" - ] - }, - "chunksize": { - "default": 16000000, - "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB", - "type": "int" - } - } - }, - "api_db": { - "name": "api_db", - "display_name": "Auto Archiver API Database", - "manifest": { - "name": "Auto Archiver API Database", - "author": "Bellingcat", - "type": [ - "database" - ], - "requires_setup": true, - "description": "\n Provides integration with the Auto Archiver API for querying and storing archival data.\n\n### Features\n- **API Integration**: Supports querying for existing archives and submitting results.\n- **Duplicate Prevention**: Avoids redundant archiving when `use_api_cache` is disabled.\n- **Configurable**: Supports settings like API endpoint, authentication token, tags, and permissions.\n- **Tagging and Metadata**: Adds tags and manages metadata for archives.\n- **Optional Storage**: Archives results conditionally based on configuration.\n\n### Setup\nRequires access to an Auto Archiver API instance and a valid API token.\n ", - "dependencies": { - "python": [ - "requests", - "loguru" - ] - }, - "entry_point": "api_db::AAApiDb", - "version": "1.0", - "configs": { - "api_endpoint": { - "required": true, - "help": "API endpoint where calls are made to" - }, - "api_token": { - "default": null, - "help": "API Bearer token." - }, - "public": { - "default": false, - "type": "bool", - "help": "whether the URL should be publicly available via the API" - }, - "author_id": { - "default": null, - "help": "which email to assign as author" - }, - "group_id": { - "default": null, - "help": "which group of users have access to the archive in case public=false as author" - }, - "use_api_cache": { - "default": false, - "type": "bool", - "help": "if True then the API database will be queried prior to any archiving operations and stop if the link has already been archived" - }, - "store_results": { - "default": true, - "type": "bool", - "help": "when set, will send the results to the API database." - }, - "tags": { - "default": [], - "help": "what tags to add to the archived URL" - } - } - }, - "configs": { - "api_endpoint": { - "required": true, - "help": "API endpoint where calls are made to" - }, - "api_token": { - "default": null, - "help": "API Bearer token." - }, - "public": { - "default": false, - "type": "bool", - "help": "whether the URL should be publicly available via the API" - }, - "author_id": { - "default": null, - "help": "which email to assign as author" - }, - "group_id": { - "default": null, - "help": "which group of users have access to the archive in case public=false as author" - }, - "use_api_cache": { - "default": false, - "type": "bool", - "help": "if True then the API database will be queried prior to any archiving operations and stop if the link has already been archived" - }, - "store_results": { - "default": true, - "type": "bool", - "help": "when set, will send the results to the API database." - }, - "tags": { - "default": [], - "help": "what tags to add to the archived URL" - } - } - }, - "console_db": { - "name": "console_db", - "display_name": "Console Database", - "manifest": { - "name": "Console Database", - "author": "Bellingcat", - "type": [ - "database" - ], - "requires_setup": false, - "description": "\nProvides a simple database implementation that outputs archival results and status updates to the console.\n\n### Features\n- Logs the status of archival tasks directly to the console, including:\n - started\n - failed (with error details)\n - aborted\n - done (with optional caching status)\n- Useful for debugging or lightweight setups where no external database is required.\n\n### Setup\nNo additional configuration is required.\n", - "dependencies": { - "python": [ - "loguru" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": {} - }, - "configs": null - }, - "csv_db": { - "name": "csv_db", - "display_name": "CSV Database", - "manifest": { - "name": "CSV Database", - "author": "Bellingcat", - "type": [ - "database" - ], - "requires_setup": false, - "description": "\nHandles exporting archival results to a CSV file.\n\n### Features\n- Saves archival metadata as rows in a CSV file.\n- Automatically creates the CSV file with a header if it does not exist.\n- Appends new metadata entries to the existing file.\n\n### Setup\nRequired config:\n- csv_file: Path to the CSV file where results will be stored (default: \"db.csv\").\n", - "dependencies": { - "python": [ - "loguru" - ] - }, - "entry_point": "csv_db::CSVDb", - "version": "1.0", - "configs": { - "csv_file": { - "default": "db.csv", - "help": "CSV file name to save metadata to" - } - } - }, - "configs": { - "csv_file": { - "default": "db.csv", - "help": "CSV file name to save metadata to" - } - } - }, - "gdrive_storage": { - "name": "gdrive_storage", - "display_name": "Google Drive Storage", - "manifest": { - "name": "Google Drive Storage", - "author": "Dave Mateer", - "type": [ - "storage" - ], - "requires_setup": true, - "description": "\n \n GDriveStorage: A storage module for saving archived content to Google Drive.\n\n Source Documentation: https://davemateer.com/2022/04/28/google-drive-with-python\n\n ### Features\n - Saves media files to Google Drive, organizing them into folders based on the provided path structure.\n - Supports OAuth token-based authentication or service account credentials for API access.\n - Automatically creates folders in Google Drive if they don't exist.\n - Retrieves CDN URLs for stored files, enabling easy sharing and access.\n\n ### Notes\n - Requires setup with either a Google OAuth token or a service account JSON file.\n - Files are uploaded to the specified `root_folder_id` and organized by the `media.key` structure.\n - Automatically handles Google Drive API token refreshes for long-running jobs.\n \n ## Overview\nThis module integrates Google Drive as a storage backend, enabling automatic folder creation and file uploads. It supports authentication via **service accounts** (recommended for automation) or **OAuth tokens** (for user-based authentication).\n\n## Features\n- Saves files to Google Drive, organizing them into structured folders.\n- Supports both **service account** and **OAuth token** authentication.\n- Automatically creates folders if they don't exist.\n- Generates public URLs for easy file sharing.\n\n## Setup Guide\n1. **Enable Google Drive API**\n - Create a Google Cloud project at [Google Cloud Console](https://console.cloud.google.com/)\n - Enable the **Google Drive API**.\n\n2. **Set Up a Google Drive Folder**\n - Create a folder in **Google Drive** and copy its **folder ID** from the URL.\n - Add the **folder ID** to your configuration (`orchestration.yaml`):\n ```yaml\n root_folder_id: \"FOLDER_ID\"\n ```\n\n3. **Authentication Options**\n - **Option 1: Service Account (Recommended)**\n - Create a **service account** in Google Cloud IAM.\n - Download the JSON key file and save it as:\n ```\n secrets/service_account.json\n ```\n - **Share your Drive folder** with the service account\u2019s `client_email` (found in the JSON file).\n \n - **Option 2: OAuth Token (User Authentication)**\n - Create OAuth **Desktop App credentials** in Google Cloud.\n - Save the credentials as:\n ```\n secrets/oauth_credentials.json\n ```\n - Generate an OAuth token by running:\n ```sh\n python scripts/create_update_gdrive_oauth_token.py -c secrets/oauth_credentials.json\n ```\n\n \n Notes on the OAuth token:\n Tokens are refreshed after 1 hour however keep working for 7 days (tbc)\n so as long as the job doesn't last for 7 days then this method of refreshing only once per run will work\n see this link for details on the token:\n https://davemateer.com/2022/04/28/google-drive-with-python#tokens\n \n \n", - "dependencies": { - "python": [ - "loguru", - "googleapiclient", - "google" - ] - }, - "entry_point": "gdrive_storage::GDriveStorage", - "version": "1.0", - "configs": { - "path_generator": { - "default": "url", - "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", - "choices": [ - "flat", - "url", - "random" - ] - }, - "filename_generator": { - "default": "static", - "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled).", - "choices": [ - "random", - "static" - ] - }, - "root_folder_id": { - "required": true, - "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'" - }, - "oauth_token": { - "default": null, - "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account." - }, - "service_account": { - "default": "secrets/service_account.json", - "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account." - } - } - }, - "configs": { - "path_generator": { - "default": "url", - "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", - "choices": [ - "flat", - "url", - "random" - ] - }, - "filename_generator": { - "default": "static", - "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled).", - "choices": [ - "random", - "static" - ] - }, - "root_folder_id": { - "required": true, - "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'" - }, - "oauth_token": { - "default": null, - "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account." - }, - "service_account": { - "default": "secrets/service_account.json", - "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account." - } - } - }, - "s3_storage": { - "name": "s3_storage", - "display_name": "S3 Storage", - "manifest": { - "name": "S3 Storage", - "author": "Bellingcat", - "type": [ - "storage" - ], - "requires_setup": true, - "description": "\n S3Storage: A storage module for saving media files to an S3-compatible object storage.\n\n ### Features\n - Uploads media files to an S3 bucket with customizable configurations.\n - Supports `random_no_duplicate` mode to avoid duplicate uploads by checking existing files based on SHA-256 hashes.\n - Automatically generates unique paths for files when duplicates are found.\n - Configurable endpoint and CDN URL for different S3-compatible providers.\n - Supports both private and public file storage, with public files being readable online.\n\n ### Notes\n - Requires S3 credentials (API key and secret) and a bucket name to function.\n - The `random_no_duplicate` option ensures no duplicate uploads by leveraging hash-based folder structures.\n - Uses `boto3` for interaction with the S3 API.\n - Depends on the `HashEnricher` module for hash calculation.\n ", - "dependencies": { - "python": [ - "hash_enricher", - "boto3", - "loguru" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "path_generator": { - "default": "flat", - "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", - "choices": [ - "flat", - "url", - "random" - ] - }, - "filename_generator": { - "default": "static", - "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled).", - "choices": [ - "random", - "static" - ] - }, - "bucket": { - "default": null, - "help": "S3 bucket name" - }, - "region": { - "default": null, - "help": "S3 region name" - }, - "key": { - "default": null, - "help": "S3 API key" - }, - "secret": { - "default": null, - "help": "S3 API secret" - }, - "random_no_duplicate": { - "default": false, - "type": "bool", - "help": "if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `no-dups/`" - }, - "endpoint_url": { - "default": "https://{region}.digitaloceanspaces.com", - "help": "S3 bucket endpoint, {region} are inserted at runtime" - }, - "cdn_url": { - "default": "https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}", - "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime" - }, - "private": { - "default": false, - "type": "bool", - "help": "if true S3 files will not be readable online" - } - } - }, - "configs": { - "path_generator": { - "default": "flat", - "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", - "choices": [ - "flat", - "url", - "random" - ] - }, - "filename_generator": { - "default": "static", - "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled).", - "choices": [ - "random", - "static" - ] - }, - "bucket": { - "default": null, - "help": "S3 bucket name" - }, - "region": { - "default": null, - "help": "S3 region name" - }, - "key": { - "default": null, - "help": "S3 API key" - }, - "secret": { - "default": null, - "help": "S3 API secret" - }, - "random_no_duplicate": { - "default": false, - "type": "bool", - "help": "if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `no-dups/`" - }, - "endpoint_url": { - "default": "https://{region}.digitaloceanspaces.com", - "help": "S3 bucket endpoint, {region} are inserted at runtime" - }, - "cdn_url": { - "default": "https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}", - "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime" - }, - "private": { - "default": false, - "type": "bool", - "help": "if true S3 files will not be readable online" - } - } - }, - "local_storage": { - "name": "local_storage", - "display_name": "Local Storage", - "manifest": { - "name": "Local Storage", - "author": "Bellingcat", - "type": [ - "storage" - ], - "requires_setup": false, - "description": "\n LocalStorage: A storage module for saving archived content locally on the filesystem.\n\n ### Features\n - Saves archived media files to a specified folder on the local filesystem.\n - Maintains file metadata during storage using `shutil.copy2`.\n - Supports both absolute and relative paths for stored files, configurable via `save_absolute`.\n - Automatically creates directories as needed for storing files.\n\n ### Notes\n - Default storage folder is `./archived`, but this can be changed via the `save_to` configuration.\n - The `save_absolute` option can reveal the file structure in output formats; use with caution.\n ", - "dependencies": { - "python": [ - "loguru" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "path_generator": { - "default": "flat", - "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", - "choices": [ - "flat", - "url", - "random" - ] - }, - "filename_generator": { - "default": "static", - "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled)", - "choices": [ - "random", - "static" - ] - }, - "save_to": { - "default": "./local_archive", - "help": "folder where to save archived content" - }, - "save_absolute": { - "default": false, - "type": "bool", - "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)" - } - } - }, - "configs": { - "path_generator": { - "default": "flat", - "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", - "choices": [ - "flat", - "url", - "random" - ] - }, - "filename_generator": { - "default": "static", - "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled)", - "choices": [ - "random", - "static" - ] - }, - "save_to": { - "default": "./local_archive", - "help": "folder where to save archived content" - }, - "save_absolute": { - "default": false, - "type": "bool", - "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)" - } - } - }, - "mute_formatter": { - "name": "mute_formatter", - "display_name": "Mute Formatter", - "manifest": { - "name": "Mute Formatter", - "author": "Bellingcat", - "type": [ - "formatter" - ], - "requires_setup": true, - "description": " Default formatter.\n ", - "dependencies": {}, - "entry_point": "", - "version": "1.0", - "configs": {} - }, - "configs": null - }, - "html_formatter": { - "name": "html_formatter", - "display_name": "HTML Formatter", - "manifest": { - "name": "HTML Formatter", - "author": "Bellingcat", - "type": [ - "formatter" - ], - "requires_setup": false, - "description": " ", - "dependencies": { - "python": [ - "hash_enricher", - "loguru", - "jinja2" - ], - "bin": [ - "" - ] - }, - "entry_point": "", - "version": "1.0", - "configs": { - "detect_thumbnails": { - "default": true, - "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'", - "type": "bool" - } - } - }, - "configs": { - "detect_thumbnails": { - "default": true, - "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'", - "type": "bool" - } - } - } - }, - "steps": { - "feeders": [ - "cli_feeder", - "atlos_feeder_db_storage", - "csv_feeder", - "gsheet_feeder_db" - ], - "extractors": [ - "wayback_extractor_enricher", - "wacz_extractor_enricher", - "instagram_api_extractor", - "instagram_tbot_extractor", - "generic_extractor", - "tiktok_tikwm_extractor", - "twitter_api_extractor", - "instagram_extractor", - "telethon_extractor", - "vk_extractor", - "telegram_extractor" - ], - "enrichers": [ - "wayback_extractor_enricher", - "wacz_extractor_enricher", - "metadata_enricher", - "timestamping_enricher", - "thumbnail_enricher", - "screenshot_enricher", - "meta_enricher", - "pdq_hash_enricher", - "whisper_enricher", - "opentimestamps_enricher", - "ssl_enricher", - "hash_enricher" - ], - "databases": [ - "console_db", - "api_db", - "csv_db", - "atlos_feeder_db_storage", - "gsheet_feeder_db" - ], - "storages": [ - "local_storage", - "gdrive_storage", - "atlos_feeder_db_storage", - "s3_storage" - ], - "formatters": [ - "html_formatter", - "mute_formatter" - ] - }, - "configs": [ - "atlos_feeder_db_storage", - "csv_feeder", - "gsheet_feeder_db", - "cli_feeder", - "instagram_api_extractor", - "instagram_tbot_extractor", - "twitter_api_extractor", - "instagram_extractor", - "telethon_extractor", - "vk_extractor", - "generic_extractor", - "wayback_extractor_enricher", - "wacz_extractor_enricher", - "timestamping_enricher", - "screenshot_enricher", - "whisper_enricher", - "opentimestamps_enricher", - "thumbnail_enricher", - "ssl_enricher", - "hash_enricher", - "api_db", - "csv_db", - "gdrive_storage", - "s3_storage", - "local_storage", - "html_formatter" - ], - "module_types": [ - "feeder", - "extractor", - "enricher", - "database", - "storage", - "formatter" - ], - "empty_config": "# Auto Archiver Configuration\n\n# Steps are the modules that will be run in the order they are defined\nsteps:\n feeders: []\n extractors: []\n enrichers: []\n databases: []\n storages: []\n formatters: []\n\n# Global configuration\n\n# Authentication\n# a dictionary of authentication information that can be used by extractors to login to website. \n# you can use a comma separated list for multiple domains on the same line (common usecase: x.com,twitter.com)\n# Common login 'types' are username/password, cookie, api key/token.\n# There are two special keys for using cookies, they are: cookies_file and cookies_from_browser. \n# Some Examples:\n# facebook.com:\n# username: \"my_username\"\n# password: \"my_password\"\n# or for a site that uses an API key:\n# twitter.com,x.com:\n# api_key\n# api_secret\n# youtube.com:\n# cookie: \"login_cookie=value ; other_cookie=123\" # multiple 'key=value' pairs should be separated by ;\n\nauthentication: {}\n\n# These are the global configurations that are used by the modules\n\nlogging:\n level: INFO\n\n" -} \ No newline at end of file diff --git a/scripts/settings/vite.config.ts b/scripts/settings/vite.config.ts index a04d8c7..67726a6 100644 --- a/scripts/settings/vite.config.ts +++ b/scripts/settings/vite.config.ts @@ -6,7 +6,7 @@ import { viteSingleFile } from "vite-plugin-singlefile" export default defineConfig({ plugins: [react(), viteSingleFile()], build: { - minify: false, - sourcemap: true, + // minify: false, + // sourcemap: true, } }); diff --git a/src/auto_archiver/core/config.py b/src/auto_archiver/core/config.py index 59c1eec..a2d7679 100644 --- a/src/auto_archiver/core/config.py +++ b/src/auto_archiver/core/config.py @@ -8,6 +8,7 @@ flexible setup in various environments. import argparse from ruamel.yaml import YAML, CommentedMap import json +import os from loguru import logger @@ -230,6 +231,10 @@ def read_yaml(yaml_filename: str) -> CommentedMap: def store_yaml(config: CommentedMap, yaml_filename: str) -> None: config_to_save = deepcopy(config) + ## if the save path is the default location (secrets) then create the 'secrets' folder + if os.path.dirname(yaml_filename) == "secrets": + os.makedirs("secrets", exist_ok=True) + auth_dict = config_to_save.get("authentication", {}) if auth_dict and auth_dict.get("load_from_file"): # remove all other values from the config, don't want to store it in the config file diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py index 672994a..d06c287 100644 --- a/src/auto_archiver/core/orchestrator.py +++ b/src/auto_archiver/core/orchestrator.py @@ -112,7 +112,7 @@ class ArchivingOrchestrator: def check_steps(self, config): for module_type in MODULE_TYPES: if not config["steps"].get(f"{module_type}s", []): - if module_type == "feeder" or module_type == "formatter" and config["steps"].get(f"{module_type}"): + if (module_type == "feeder" or module_type == "formatter") and config["steps"].get(f"{module_type}"): raise SetupError( f"It appears you have '{module_type}' set under 'steps' in your configuration file, but as of version 0.13.0 of Auto Archiver, you must use '{module_type}s'. Change this in your configuration file and try again. \ Here's how that would look: \n\nsteps:\n {module_type}s:\n - [your_{module_type}_name_here]\n {'extractors:...' if module_type == 'feeder' else '...'}\n" @@ -377,7 +377,8 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_ try: loaded_module: BaseModule = self.module_factory.get_module(module, self.config) except (KeyboardInterrupt, Exception) as e: - logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}") + if not isinstance(e, KeyboardInterrupt) and not isinstance(e, SetupError): + logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}") if loaded_module and module_type == "extractor": loaded_module.cleanup() raise e diff --git a/src/auto_archiver/modules/cli_feeder/cli_feeder.py b/src/auto_archiver/modules/cli_feeder/cli_feeder.py index 4367bbc..5935466 100644 --- a/src/auto_archiver/modules/cli_feeder/cli_feeder.py +++ b/src/auto_archiver/modules/cli_feeder/cli_feeder.py @@ -2,13 +2,14 @@ from loguru import logger from auto_archiver.core.feeder import Feeder from auto_archiver.core.metadata import Metadata +from auto_archiver.core.consts import SetupError class CLIFeeder(Feeder): def setup(self) -> None: self.urls = self.config["urls"] if not self.urls: - raise ValueError( + raise SetupError( "No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information." ) diff --git a/src/auto_archiver/modules/generic_extractor/__manifest__.py b/src/auto_archiver/modules/generic_extractor/__manifest__.py index 4c57667..9ef1cb3 100644 --- a/src/auto_archiver/modules/generic_extractor/__manifest__.py +++ b/src/auto_archiver/modules/generic_extractor/__manifest__.py @@ -15,6 +15,9 @@ supported by `yt-dlp`, such as YouTube, Facebook, and others. It provides functi for retrieving videos, subtitles, comments, and other metadata, and it integrates with the broader archiving framework. +For a full list of video platforms supported by `yt-dlp`, see the +[official documentation](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md) + ### Features - Supports downloading videos and playlists. - Retrieves metadata like titles, descriptions, upload dates, and durations. diff --git a/src/auto_archiver/modules/generic_extractor/dropin.py b/src/auto_archiver/modules/generic_extractor/dropin.py index 723c8fc..8395f09 100644 --- a/src/auto_archiver/modules/generic_extractor/dropin.py +++ b/src/auto_archiver/modules/generic_extractor/dropin.py @@ -1,3 +1,4 @@ +from typing import Type from yt_dlp.extractor.common import InfoExtractor from auto_archiver.core.metadata import Metadata from auto_archiver.core.extractor import Extractor @@ -24,6 +25,8 @@ class GenericDropin: """ + extractor: Type[Extractor] = None + def extract_post(self, url: str, ie_instance: InfoExtractor): """ This method should return the post data from the url. @@ -55,3 +58,10 @@ class GenericDropin: This method should download any additional media from the post. """ return metadata + + def is_suitable(self, url, info_extractor: InfoExtractor): + """ + Used to override the InfoExtractor's 'is_suitable' method. Dropins should override this method to return True if the url is suitable for the extractor + (based on being able to parse other URLs) + """ + return False diff --git a/src/auto_archiver/modules/generic_extractor/facebook.py b/src/auto_archiver/modules/generic_extractor/facebook.py index 36c5e60..e04a862 100644 --- a/src/auto_archiver/modules/generic_extractor/facebook.py +++ b/src/auto_archiver/modules/generic_extractor/facebook.py @@ -1,17 +1,154 @@ +import re from .dropin import GenericDropin +from auto_archiver.core.metadata import Metadata +from yt_dlp.extractor.facebook import FacebookIE + +# TODO: Remove if / when https://github.com/yt-dlp/yt-dlp/pull/12275 is merged +from yt_dlp.utils import ( + clean_html, + get_element_by_id, + traverse_obj, + get_first, + merge_dicts, + int_or_none, + parse_count, +) + + +def _extract_metadata(self, webpage, video_id): + post_data = [ + self._parse_json(j, video_id, fatal=False) + for j in re.findall(r"data-sjs>({.*?ScheduledServerJS.*?})", webpage) + ] + post = ( + traverse_obj( + post_data, + (..., "require", ..., ..., ..., "__bbox", "require", ..., ..., ..., "__bbox", "result", "data"), + expected_type=dict, + ) + or [] + ) + media = traverse_obj( + post, + ( + ..., + "attachments", + ..., + lambda k, v: (k == "media" and str(v["id"]) == video_id and v["__typename"] == "Video"), + ), + expected_type=dict, + ) + title = get_first(media, ("title", "text")) + description = get_first(media, ("creation_story", "comet_sections", "message", "story", "message", "text")) + page_title = title or self._html_search_regex( + ( + r']*class="uiHeaderTitle"[^>]*>(?P[^<]*)', + r'(?s)(?P.*?)', + self._meta_regex("og:title"), + self._meta_regex("twitter:title"), + r"(?P<content>.+?)", + ), + webpage, + "title", + default=None, + group="content", + ) + description = description or self._html_search_meta( + ["description", "og:description", "twitter:description"], webpage, "description", default=None + ) + uploader_data = ( + get_first(media, ("owner", {dict})) + or get_first( + post, ("video", "creation_story", "attachments", ..., "media", lambda k, v: k == "owner" and v["name"]) + ) + or get_first(post, (..., "video", lambda k, v: k == "owner" and v["name"])) + or get_first(post, ("node", "actors", ..., {dict})) + or get_first(post, ("event", "event_creator", {dict})) + or get_first(post, ("video", "creation_story", "short_form_video_context", "video_owner", {dict})) + or {} + ) + uploader = uploader_data.get("name") or ( + clean_html(get_element_by_id("fbPhotoPageAuthorName", webpage)) + or self._search_regex( + (r'ownerName\s*:\s*"([^"]+)"', *self._og_regexes("title")), webpage, "uploader", fatal=False + ) + ) + timestamp = int_or_none(self._search_regex(r']+data-utime=["\'](\d+)', webpage, "timestamp", default=None)) + thumbnail = self._html_search_meta(["og:image", "twitter:image"], webpage, "thumbnail", default=None) + # some webpages contain unretrievable thumbnail urls + # like https://lookaside.fbsbx.com/lookaside/crawler/media/?media_id=10155168902769113&get_thumbnail=1 + # in https://www.facebook.com/yaroslav.korpan/videos/1417995061575415/ + if thumbnail and not re.search(r"\.(?:jpg|png)", thumbnail): + thumbnail = None + info_dict = { + "description": description, + "uploader": uploader, + "uploader_id": uploader_data.get("id"), + "timestamp": timestamp, + "thumbnail": thumbnail, + "view_count": parse_count( + self._search_regex( + (r'\bviewCount\s*:\s*["\']([\d,.]+)', r'video_view_count["\']\s*:\s*(\d+)'), + webpage, + "view count", + default=None, + ) + ), + "concurrent_view_count": get_first( + post, (("video", (..., ..., "attachments", ..., "media")), "liveViewerCount", {int_or_none}) + ), + **traverse_obj( + post, + ( + lambda _, v: video_id in v["url"], + "feedback", + { + "like_count": ("likers", "count", {int}), + "comment_count": ("total_comment_count", {int}), + "repost_count": ("share_count_reduced", {parse_count}), + }, + ), + get_all=False, + ), + } + + info_json_ld = self._search_json_ld(webpage, video_id, default={}) + info_json_ld["title"] = ( + re.sub(r"\s*\|\s*Facebook$", "", title or info_json_ld.get("title") or page_title or "") + or (description or "").replace("\n", " ") + or f"Facebook video #{video_id}" + ) + return merge_dicts(info_json_ld, info_dict) class Facebook(GenericDropin): - def extract_post(self, url: str, ie_instance): - video_id = ie_instance._match_valid_url(url).group("id") - ie_instance._download_webpage(url.replace("://m.facebook.com/", "://www.facebook.com/"), video_id) - webpage = ie_instance._download_webpage(url, ie_instance._match_valid_url(url).group("id")) + def extract_post(self, url: str, ie_instance: FacebookIE): + post_id_regex = r"(?Ppfbid[A-Za-z0-9]+|\d+|t\.(\d+\/\d+))" + post_id = re.search(post_id_regex, url).group("id") + webpage = ie_instance._download_webpage(url.replace("://m.facebook.com/", "://www.facebook.com/"), post_id) - # TODO: fix once https://github.com/yt-dlp/yt-dlp/pull/12275 is merged - post_data = ie_instance._extract_metadata(webpage) + # TODO: For long posts, this _extract_metadata only seems to return the first 100 or so characters, followed by ... + + # TODO: If/when https://github.com/yt-dlp/yt-dlp/pull/12275 is merged, uncomment next line and delete the one after + # post_data = ie_instance._extract_metadata(webpage, post_id) + post_data = _extract_metadata(ie_instance, webpage, post_id) return post_data - def create_metadata(self, post: dict, ie_instance, archiver, url): - metadata = archiver.create_metadata(url) - metadata.set_title(post.get("title")).set_content(post.get("description")).set_post_data(post) - return metadata + def create_metadata(self, post: dict, ie_instance: FacebookIE, archiver, url): + result = Metadata() + result.set_content(post.get("description", "")) + result.set_title(post.get("title", "")) + result.set("author", post.get("uploader", "")) + result.set_url(url) + return result + + def is_suitable(self, url, info_extractor: FacebookIE): + regex = r"(?:https?://(?:[\w-]+\.)?(?:facebook\.com||facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd\.onion)/)" + return re.match(regex, url) + + def skip_ytdlp_download(self, url: str, is_instance: FacebookIE): + """ + Skip using the ytdlp download method for Facebook *photo* posts, they have a URL with an id of t.XXXXX/XXXXX + """ + if re.search(r"/t.\d+/\d+", url): + return True diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py index 4a54759..e96ac41 100644 --- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py +++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py @@ -67,8 +67,18 @@ class GenericExtractor(Extractor): """ Returns a list of valid extractors for the given URL""" for info_extractor in yt_dlp.YoutubeDL()._ies.values(): - if info_extractor.suitable(url) and info_extractor.working(): + if not info_extractor.working(): + continue + + # check if there's a dropin and see if that declares whether it's suitable + dropin = self.dropin_for_name(info_extractor.ie_key()) + if dropin and dropin.is_suitable(url, info_extractor): yield info_extractor + continue + + if info_extractor.suitable(url): + yield info_extractor + continue def suitable(self, url: str) -> bool: """ @@ -188,9 +198,13 @@ class GenericExtractor(Extractor): result = self.download_additional_media(video_data, info_extractor, result) # keep both 'title' and 'fulltitle', but prefer 'title', falling back to 'fulltitle' if it doesn't exist - result.set_title(video_data.pop("title", video_data.pop("fulltitle", ""))) - result.set_url(url) - if "description" in video_data: + if not result.get_title(): + result.set_title(video_data.pop("title", video_data.pop("fulltitle", ""))) + + if not result.get("url"): + result.set_url(url) + + if "description" in video_data and not result.get_content(): result.set_content(video_data["description"]) # extract comments if enabled if self.comments: @@ -207,10 +221,10 @@ class GenericExtractor(Extractor): ) # then add the common metadata - if timestamp := video_data.pop("timestamp", None): + if timestamp := video_data.pop("timestamp", None) and not result.get("timestamp"): timestamp = datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).isoformat() result.set_timestamp(timestamp) - if upload_date := video_data.pop("upload_date", None): + if upload_date := video_data.pop("upload_date", None) and not result.get("upload_date"): upload_date = datetime.datetime.strptime(upload_date, "%Y%m%d").replace(tzinfo=datetime.timezone.utc) result.set("upload_date", upload_date) @@ -240,7 +254,8 @@ class GenericExtractor(Extractor): return False post_data = dropin.extract_post(url, ie_instance) - return dropin.create_metadata(post_data, ie_instance, self, url) + result = dropin.create_metadata(post_data, ie_instance, self, url) + return self.add_metadata(post_data, info_extractor, url, result) def get_metadata_for_video( self, data: dict, info_extractor: Type[InfoExtractor], url: str, ydl: yt_dlp.YoutubeDL @@ -296,6 +311,7 @@ class GenericExtractor(Extractor): def _load_dropin(dropin): dropin_class = getattr(dropin, dropin_class_name)() + dropin.extractor = self return self._dropins.setdefault(dropin_name, dropin_class) try: @@ -340,7 +356,7 @@ class GenericExtractor(Extractor): dropin_submodule = self.dropin_for_name(info_extractor.ie_key()) try: - if dropin_submodule and dropin_submodule.skip_ytdlp_download(info_extractor, url): + if dropin_submodule and dropin_submodule.skip_ytdlp_download(url, info_extractor): logger.debug(f"Skipping using ytdlp to download files for {info_extractor.ie_key()}") raise SkipYtdlp() @@ -359,7 +375,7 @@ class GenericExtractor(Extractor): if not isinstance(e, SkipYtdlp): logger.debug( - f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use extractor to get post data instead' + f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use dropin to get post data instead' ) try: diff --git a/src/auto_archiver/modules/generic_extractor/tiktok.py b/src/auto_archiver/modules/generic_extractor/tiktok.py index e05d298..b25abca 100644 --- a/src/auto_archiver/modules/generic_extractor/tiktok.py +++ b/src/auto_archiver/modules/generic_extractor/tiktok.py @@ -38,6 +38,9 @@ class Tiktok(GenericDropin): api_data["video_url"] = video_url return api_data + def keys_to_clean(self, video_data: dict, info_extractor): + return ["video_url", "title", "create_time", "author", "cover", "origin_cover", "ai_dynamic_cover", "duration"] + def create_metadata(self, post: dict, ie_instance, archiver, url): # prepare result, start by downloading video result = Metadata() @@ -54,17 +57,17 @@ class Tiktok(GenericDropin): logger.error(f"failed to download video from {video_url}") return False video_media = Media(video_downloaded) - if duration := post.pop("duration", None): + if duration := post.get("duration", None): video_media.set("duration", duration) result.add_media(video_media) # add remaining metadata - result.set_title(post.pop("title", "")) + result.set_title(post.get("title", "")) - if created_at := post.pop("create_time", None): + if created_at := post.get("create_time", None): result.set_timestamp(datetime.fromtimestamp(created_at, tz=timezone.utc)) - if author := post.pop("author", None): + if author := post.get("author", None): result.set("author", author) result.set("api_data", post) diff --git a/src/auto_archiver/modules/local_storage/__manifest__.py b/src/auto_archiver/modules/local_storage/__manifest__.py index b5307b8..3a7f481 100644 --- a/src/auto_archiver/modules/local_storage/__manifest__.py +++ b/src/auto_archiver/modules/local_storage/__manifest__.py @@ -20,7 +20,7 @@ "save_absolute": { "default": False, "type": "bool", - "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)", + "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (Warning: saving an absolute path will show your computer's file structure)", }, }, "description": """ diff --git a/src/auto_archiver/utils/webdriver.py b/src/auto_archiver/utils/webdriver.py index c866c25..43e7817 100644 --- a/src/auto_archiver/utils/webdriver.py +++ b/src/auto_archiver/utils/webdriver.py @@ -49,7 +49,7 @@ class CookieSettingDriver(webdriver.Firefox): self.driver.add_cookie({"name": name, "value": value}) elif self.cookiejar: domain = urlparse(url).netloc - regex = re.compile(f"(www)?\.?{domain}$") + regex = re.compile(f"(www)?.?{domain}$") for cookie in self.cookiejar: if regex.match(cookie.domain): try: diff --git a/tests/extractors/test_generic_extractor.py b/tests/extractors/test_generic_extractor.py index ec7aae4..2089007 100644 --- a/tests/extractors/test_generic_extractor.py +++ b/tests/extractors/test_generic_extractor.py @@ -40,6 +40,22 @@ class TestGenericExtractor(TestExtractorBase): path = os.path.join(dirname(dirname(__file__)), "data/") assert self.extractor.dropin_for_name("dropin", additional_paths=[path]) + @pytest.mark.parametrize( + "url, suitable_extractors", + [ + ("https://www.youtube.com/watch?v=5qap5aO4i9A", ["youtube"]), + ("https://www.tiktok.com/@funnycats0ftiktok/video/7345101300750748970?lang=en", ["tiktok"]), + ("https://www.instagram.com/p/CU1J9JYJ9Zz/", ["instagram"]), + ("https://www.facebook.com/nytimes/videos/10160796550110716", ["facebook"]), + ("https://www.facebook.com/BylineFest/photos/t.100057299682816/927879487315946/", ["facebook"]), + ], + ) + def test_suitable_extractors(self, url, suitable_extractors): + suitable_extractors = suitable_extractors + ["generic"] # the generic is valid for all + extractors = list(self.extractor.suitable_extractors(url)) + assert len(extractors) == len(suitable_extractors) + assert [e.ie_key().lower() for e in extractors] == suitable_extractors + @pytest.mark.parametrize( "url, is_suitable", [ @@ -55,7 +71,7 @@ class TestGenericExtractor(TestExtractorBase): ("https://google.com", True), ], ) - def test_suitable_urls(self, make_item, url, is_suitable): + def test_suitable_urls(self, url, is_suitable): """ Note: expected behaviour is to return True for all URLs, as YoutubeDLArchiver should be able to handle all URLs This behaviour may be changed in the future (e.g. if we want the youtubedl archiver to just handle URLs it has extractors for, @@ -245,3 +261,32 @@ class TestGenericExtractor(TestExtractorBase): self.assertValidResponseMetadata(post, title, timestamp) assert len(post.media) == 1 assert post.media[0].hash == image_hash + + @pytest.mark.download + def test_download_facebook_video(self, make_item): + post = self.extractor.download(make_item("https://www.facebook.com/bellingcat/videos/588371253839133")) + assert len(post.media) == 2 + assert post.media[0].filename.endswith("588371253839133.mp4") + assert post.media[0].mimetype == "video/mp4" + + assert post.media[1].filename.endswith(".jpg") + assert post.media[1].mimetype == "image/jpeg" + + assert "Bellingchat Premium is with Kolina Koltai" in post.get_title() + + @pytest.mark.download + def test_download_facebook_image(self, make_item): + post = self.extractor.download( + make_item("https://www.facebook.com/BylineFest/photos/t.100057299682816/927879487315946/") + ) + + assert len(post.media) == 1 + assert post.media[0].filename.endswith(".png") + assert "Byline Festival - BylineFest Partner" == post.get_title() + + @pytest.mark.download + def test_download_facebook_text_only(self, make_item): + url = "https://www.facebook.com/bellingcat/posts/pfbid02rzpwZxAZ8bLkAX8NvHv4DWAidFaqAUfJMbo9vWkpwxL7uMUWzWMiizXLWRSjwihVl" + post = self.extractor.download(make_item(url)) + assert "Bellingcat researcher Kolina Koltai delves deeper into Clothoff" in post.get("content") + assert post.get_title() == "Bellingcat"