[YouTube] Fix crash on SABR-only player responses, do not use WEB client

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java

@@ -11,12 +11,6 @@ import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_OS_VERSION;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.MOBILE_CLIENT_PLATFORM;
-import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_ID;
-import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_NAME;
-import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_PLATFORM;
-import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION;
-import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MAKE;
-import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MODEL_AND_OS_NAME;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WATCH_CLIENT_SCREEN;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME;
@@ -118,16 +112,6 @@ public final class InnertubeClientRequestInfo {
                         null, null, -1));
     }
 
-    @Nonnull
-    public static InnertubeClientRequestInfo ofTvHtml5Client() {
-        return new InnertubeClientRequestInfo(
-                new InnertubeClientRequestInfo.ClientInfo(TVHTML5_CLIENT_NAME,
-                        TVHTML5_CLIENT_VERSION, WATCH_CLIENT_SCREEN, TVHTML5_CLIENT_ID, null),
-                new InnertubeClientRequestInfo.DeviceInfo(TVHTML5_CLIENT_PLATFORM,
-                        TVHTML5_DEVICE_MAKE, TVHTML5_DEVICE_MODEL_AND_OS_NAME,
-                        TVHTML5_DEVICE_MODEL_AND_OS_NAME, "", -1));
-    }
-
     @Nonnull
     public static InnertubeClientRequestInfo ofAndroidClient() {
         return new InnertubeClientRequestInfo(

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java

@@ -17,9 +17,6 @@ import java.util.List;
 import java.util.Map;
 
 import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
-import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_ID;
-import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION;
-import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_VERSION;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK;
@@ -82,77 +79,6 @@ public final class YoutubeStreamHelper {
                         url, headers, body, localization)));
     }
 
-    @Nonnull
-    public static JsonObject getTvHtml5PlayerResponse(
-            @Nonnull final Localization localization,
-            @Nonnull final ContentCountry contentCountry,
-            @Nonnull final String videoId,
-            @Nonnull final String cpn,
-            final int signatureTimestamp) throws IOException, ExtractionException {
-        final InnertubeClientRequestInfo innertubeClientRequestInfo =
-                InnertubeClientRequestInfo.ofTvHtml5Client();
-
-        final Map<String, List<String>> headers = new HashMap<>(
-                getClientHeaders(TVHTML5_CLIENT_ID, TVHTML5_CLIENT_VERSION));
-        headers.putAll(getOriginReferrerHeaders("https://www.youtube.com"));
-        headers.put("User-Agent", List.of(TVHTML5_USER_AGENT));
-
-        // We must always pass a valid visitorData to get valid player responses, which needs to be
-        // got from YouTube
-        // For some reason, the TVHTML5 client doesn't support the visitor_id endpoint, use the
-        // guide one instead, which is quite lightweight
-        innertubeClientRequestInfo.clientInfo.visitorData =
-                YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo,
-                        localization, contentCountry, headers, YOUTUBEI_V1_URL, null, true);
-
-        final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
-                innertubeClientRequestInfo, null);
-
-        addVideoIdCpnAndOkChecks(builder, videoId, cpn);
-
-        addPlaybackContext(builder, BASE_YT_DESKTOP_WATCH_URL + videoId, signatureTimestamp);
-
-        final byte[] body = JsonWriter.string(builder.done())
-                .getBytes(StandardCharsets.UTF_8);
-
-        final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER;
-
-        return JsonUtils.toJsonObject(getValidJsonResponseBody(
-                getDownloader().postWithContentTypeJson(url, headers, body, localization)));
-    }
-
-    @Nonnull
-    public static JsonObject getWebFullPlayerResponse(
-            @Nonnull final Localization localization,
-            @Nonnull final ContentCountry contentCountry,
-            @Nonnull final String videoId,
-            @Nonnull final String cpn,
-            @Nonnull final PoTokenResult webPoTokenResult,
-            final int signatureTimestamp) throws IOException, ExtractionException {
-        final InnertubeClientRequestInfo innertubeClientRequestInfo =
-                InnertubeClientRequestInfo.ofWebClient();
-        innertubeClientRequestInfo.clientInfo.clientVersion = getClientVersion();
-        innertubeClientRequestInfo.clientInfo.visitorData = webPoTokenResult.visitorData;
-
-        final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
-                innertubeClientRequestInfo, null);
-
-        addVideoIdCpnAndOkChecks(builder, videoId, cpn);
-
-        addPlaybackContext(builder, BASE_YT_DESKTOP_WATCH_URL + videoId, signatureTimestamp);
-
-        addPoToken(builder, webPoTokenResult.playerRequestPoToken);
-
-        final byte[] body = JsonWriter.string(builder.done())
-                .getBytes(StandardCharsets.UTF_8);
-
-        final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER;
-
-        return JsonUtils.toJsonObject(getValidJsonResponseBody(
-                getDownloader().postWithContentTypeJson(
-                        url, getYouTubeHeaders(), body, localization)));
-    }
-
     @Nonnull
     public static JsonObject getWebEmbeddedPlayerResponse(
             @Nonnull final Localization localization,
@@ -247,11 +173,9 @@ public final class YoutubeStreamHelper {
         final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
                 innertubeClientRequestInfo, null);
 
+        builder.object("playerRequest");
         addVideoIdCpnAndOkChecks(builder, videoId, cpn);
-
+        builder.end()
-        builder.object("playerRequest")
-                .value(VIDEO_ID, videoId)
-                .end()
                 .value("disablePlayerResponse", false);
 
         final byte[] body = JsonWriter.string(builder.done())

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -813,22 +813,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         final Localization localization = getExtractorLocalization();
         final ContentCountry contentCountry = getExtractorContentCountry();
 
-        final PoTokenProvider poTokenproviderInstance = poTokenProvider;
+        final PoTokenProvider poTokenProviderInstance = poTokenProvider;
-        final boolean noPoTokenProviderSet = poTokenproviderInstance == null;
+        final boolean noPoTokenProviderSet = poTokenProviderInstance == null;
 
-        fetchHtml5Client(localization, contentCountry, videoId, poTokenproviderInstance,
+        fetchHtml5Client(localization, contentCountry, videoId, poTokenProviderInstance);
-                noPoTokenProviderSet);
 
         setStreamType();
 
         final PoTokenResult androidPoTokenResult = noPoTokenProviderSet ? null
-                : poTokenproviderInstance.getAndroidClientPoToken(videoId);
+                : poTokenProviderInstance.getAndroidClientPoToken(videoId);
 
         fetchAndroidClient(localization, contentCountry, videoId, androidPoTokenResult);
 
         if (fetchIosClient) {
             final PoTokenResult iosPoTokenResult = noPoTokenProviderSet ? null
-                    : poTokenproviderInstance.getIosClientPoToken(videoId);
+                    : poTokenProviderInstance.getIosClientPoToken(videoId);
             fetchIosClient(localization, contentCountry, videoId, iosPoTokenResult);
         }
 
@@ -904,82 +903,32 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     private void fetchHtml5Client(@Nonnull final Localization localization,
                                   @Nonnull final ContentCountry contentCountry,
                                   @Nonnull final String videoId,
-                                  @Nullable final PoTokenProvider poTokenProviderInstance,
+                                  @Nullable final PoTokenProvider poTokenProviderInstance)
-                                  final boolean noPoTokenProviderSet)
             throws IOException, ExtractionException {
         html5Cpn = generateContentPlaybackNonce();
 
-        // Suppress NPE warning as nullability is already checked before and passed with
+        final JsonObject webPlayerResponse = YoutubeStreamHelper.getWebMetadataPlayerResponse(
-        // noPoTokenProviderSet
-        //noinspection DataFlowIssue
-        final PoTokenResult webPoTokenResult = noPoTokenProviderSet ? null
-                : poTokenProviderInstance.getWebClientPoToken(videoId);
-        final JsonObject webPlayerResponse;
-        if (noPoTokenProviderSet || webPoTokenResult == null) {
-            webPlayerResponse = YoutubeStreamHelper.getWebMetadataPlayerResponse(
                     localization, contentCountry, videoId);
 
-            throwExceptionIfPlayerResponseNotValid(webPlayerResponse, videoId);
+        throwExceptionIfPlayerResponseNotValid(webPlayerResponse, videoId);
 
-            // Save the webPlayerResponse into playerResponse in the case the video cannot be
+        // Save the webPlayerResponse into playerResponse in the case the video cannot be
-            // played, so some metadata can be retrieved
+        // played, so some metadata can be retrieved
-            playerResponse = webPlayerResponse;
+        playerResponse = webPlayerResponse;
 
-            // The microformat JSON object of the content is only returned on the WEB client,
+        // The microformat JSON object of the content is only returned on the WEB client,
-            // so we need to store it instead of getting it directly from the playerResponse
+        // so we need to store it instead of getting it directly from the playerResponse
-            playerMicroFormatRenderer = playerResponse.getObject("microformat")
+        playerMicroFormatRenderer = playerResponse.getObject("microformat")
-                    .getObject("playerMicroformatRenderer");
+                .getObject("playerMicroformatRenderer");
 
-            final JsonObject playabilityStatus = webPlayerResponse.getObject(PLAYABILITY_STATUS);
+        final JsonObject playabilityStatus = webPlayerResponse.getObject(PLAYABILITY_STATUS);
 
-            if (isVideoAgeRestricted(playabilityStatus)) {
+        if (isVideoAgeRestricted(playabilityStatus)) {
-                fetchHtml5EmbedClient(localization, contentCountry, videoId,
+            fetchHtml5EmbedClient(localization, contentCountry, videoId,
-                        noPoTokenProviderSet ? null
+                    poTokenProviderInstance == null ? null
-                                : poTokenProviderInstance.getWebEmbedClientPoToken(videoId));
+                            : poTokenProviderInstance.getWebEmbedClientPoToken(videoId));
-            } else {
-                checkPlayabilityStatus(playabilityStatus);
-
-                final JsonObject tvHtml5PlayerResponse =
-                        YoutubeStreamHelper.getTvHtml5PlayerResponse(
-                                localization, contentCountry, videoId, html5Cpn,
-                                YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId));
-
-                if (isPlayerResponseNotValid(tvHtml5PlayerResponse, videoId)) {
-                    throw new ExtractionException("TVHTML5 player response is not valid");
-                }
-
-                html5StreamingData = tvHtml5PlayerResponse.getObject(STREAMING_DATA);
-                playerCaptionsTracklistRenderer = tvHtml5PlayerResponse.getObject(CAPTIONS)
-                        .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER);
-            }
         } else {
-            webPlayerResponse = YoutubeStreamHelper.getWebFullPlayerResponse(
+            checkPlayabilityStatus(playabilityStatus);
-                    localization, contentCountry, videoId, html5Cpn, webPoTokenResult,
-                    YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId));
-
-            throwExceptionIfPlayerResponseNotValid(webPlayerResponse, videoId);
-
-            // Save the webPlayerResponse into playerResponse in the case the video cannot be
-            // played, so some metadata can be retrieved
-            playerResponse = webPlayerResponse;
-
-            // The microformat JSON object of the content is only returned on the WEB client,
-            // so we need to store it instead of getting it directly from the playerResponse
-            playerMicroFormatRenderer = playerResponse.getObject("microformat")
-                    .getObject("playerMicroformatRenderer");
-
-            final JsonObject playabilityStatus = webPlayerResponse.getObject(PLAYABILITY_STATUS);
-
-            if (isVideoAgeRestricted(playabilityStatus)) {
-                fetchHtml5EmbedClient(localization, contentCountry, videoId,
-                        poTokenProviderInstance.getWebEmbedClientPoToken(videoId));
-            } else {
-                checkPlayabilityStatus(playabilityStatus);
-                html5StreamingData = webPlayerResponse.getObject(STREAMING_DATA);
-                playerCaptionsTracklistRenderer = webPlayerResponse.getObject(CAPTIONS)
-                        .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER);
-                html5StreamingUrlsPoToken = webPoTokenResult.streamingDataPoToken;
-            }
         }
     }
 
@@ -1383,6 +1332,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             // This url has an obfuscated signature
             final String cipherString = formatData.getString(CIPHER,
                     formatData.getString(SIGNATURE_CIPHER));
+
+            if (isNullOrEmpty(cipherString)) {
+                return null;
+            }
+
             final var cipher = Parser.compatParseMap(cipherString);
             final String signature = YoutubeJavaScriptPlayerManager.deobfuscateSignature(videoId,
                     cipher.getOrDefault("s", ""));