diff --git a/scripts/potoken_provider/setup_pot_provider.sh b/scripts/potoken_provider/setup_pot_provider.sh old mode 100644 new mode 100755 index e0f6d7c..85c9e60 --- a/scripts/potoken_provider/setup_pot_provider.sh +++ b/scripts/potoken_provider/setup_pot_provider.sh @@ -1,31 +1,42 @@ #!/bin/bash -set -e +set -e # Exit on error SCRIPTS_DIR="scripts/potoken_provider" -BGUTIL_DIR="$SCRIPTS_DIR/bgutil-ytdlp-pot-provider" -UPDATE=false +TARGET_DIR="$SCRIPTS_DIR/bgutil-provider" +SERVER_DIR="$TARGET_DIR/server" +GEN_SCRIPT="$SERVER_DIR/build/generate_once.js" -# Parse optional flag -if [[ "$1" == "--update" ]]; then - UPDATE=true -fi - -# Clone the repository, or update if it exists -if [ ! -d "$BGUTIL_DIR" ]; then - echo "Cloning bgutil-ytdlp-pot-provider repository..." - git clone https://github.com/Brainicism/bgutil-ytdlp-pot-provider.git "$BGUTIL_DIR" -elif [ "$UPDATE" == true ]; then - echo "Updating existing bgutil-ytdlp-pot-provider repository..." - cd "$BGUTIL_DIR" || exit 1 - git pull origin master +# Ensure the server directory exists +if [ ! -d "$SERVER_DIR" ]; then + echo "Error: PO Token provider server directory is missing! Please run update_pot_provider.sh first." + exit 1 fi # Move into the server directory -cd "$BGUTIL_DIR/server" || exit 1 +cd "$SERVER_DIR" || exit 1 -# Install dependencies and transpile the script -yarn install --frozen-lockfile -npx tsc +# Check if dependencies need installation +if [ ! -d "node_modules" ]; then + echo "Installing dependencies..." + yarn install --frozen-lockfile +else + echo "Dependencies already installed. Skipping yarn install." +fi -# The transpiled POT generation script is now available and will be used automatically by the generic extractor -echo "PO Token provider script is ready: $BGUTIL_DIR/server/build/generate_once.js" +# Check if build directory exists and if transpiling is needed +if [ ! -d "build" ] || [ "$SERVER_DIR/src" -nt "$GEN_SCRIPT" ]; then + echo "Build directory missing or outdated. Running transpilation..." + npx tsc +else + echo "Build directory is up to date. Skipping transpilation." +fi + +# Ensure the script exists after transpilation +if [ ! -f "$GEN_SCRIPT" ]; then + echo "Error: PO Token script not found after attempting transpilation." + exit 1 +fi + + +# Confirm success +echo "PO Token provider script is ready for use." diff --git a/scripts/potoken_provider/update_pot_provider.sh b/scripts/potoken_provider/update_pot_provider.sh new file mode 100755 index 0000000..58d8783 --- /dev/null +++ b/scripts/potoken_provider/update_pot_provider.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e # Exit on error + +SCRIPTS_DIR="scripts/potoken_provider" +TARGET_DIR="$SCRIPTS_DIR/bgutil-provider" +BGUTIL_REPO="https://github.com/Brainicism/bgutil-ytdlp-pot-provider.git" +BGUTIL_TEMP_DIR="$SCRIPTS_DIR/bgutil-temp" + + +# Clone fresh copy into temporary directory +git clone --depth 1 "$REPO_URL" "$TMP_DIR" + +# Ensure the target directory exists +rm -rf "$TARGET_DIR" +mkdir -p "$TARGET_DIR" + +# Copy the entire server directory +echo "Copying /server/ directory..." +cp -r "$BGUTIL_TEMP_DIR/server" "$TARGET_DIR/" + +# Clean up: remove the cloned repository +echo "Cleaning up temporary files..." +rm -rf "$BGUTIL_TEMP_DIR" + +# Confirm success +echo "PO Token provider script is ready in: $TARGET_DIR/server" +echo "Commit and push changes to include it in version control." diff --git a/src/auto_archiver/modules/generic_extractor/__manifest__.py b/src/auto_archiver/modules/generic_extractor/__manifest__.py index bd75c1c..956a924 100644 --- a/src/auto_archiver/modules/generic_extractor/__manifest__.py +++ b/src/auto_archiver/modules/generic_extractor/__manifest__.py @@ -74,8 +74,12 @@ If you are having issues with the extractor, you can review the version of `yt-d "default": "inf", "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit.", }, + "update_pots": { + "default": False, + "help": "If set, will run the script to update the pot generation script.", + }, "extractor_args": { - "default": {"youtube": {"getpot_bgutil_script": "scripts/potoken_provider/bgutil-ytdlp-pot-provider/server/build/generate_once.js"}}, + "default": {}, "help": "Additional arguments to pass to the yt-dlp extractor. See https://github.com/yt-dlp/yt-dlp/blob/master/README.md#extractor-arguments.", "type": "json_loader", }, diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py index de538d9..d46e64f 100644 --- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py +++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py @@ -73,7 +73,8 @@ class GenericExtractor(Extractor): # Determine the default location for the transpiled PO token script. default_script = os.path.join( - "scripts", "potoken_provider", "bgutil-ytdlp-pot-provider", "server", "build", "generate_once.js" + # "scripts", "potoken_provider", "bgutil-server", "build", "generate_once.js" + "scripts", "potoken_provider", "bgutil-provider", "server", "build", "generate_once.js" ) # Check if the PO token script exists. if not, trigger the script generation. if not os.path.exists(default_script): @@ -445,6 +446,8 @@ class GenericExtractor(Extractor): "--write-subs" if self.subtitles else "--no-write-subs", "--write-auto-subs" if self.subtitles else "--no-write-auto-subs", "--live-from-start" if self.live_from_start else "--no-live-from-start", + # TODO + "--verbose" ] # proxy handling