kopia lustrzana https://github.com/TeamNewPipe/NewPipeExtractor
Merge 699b408639 into 0b99100dbd
Thomas Szatmary
GitHub
commit
5e322c0d58
extractor/src
main/java/org/schabi/newpipe/extractor/services/youtube
test/java/org/schabi/newpipe/extractor/services/youtube
|
|
@ -30,6 +30,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsE
|
|||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicTrendingExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
|
||||
|
|
@ -39,6 +40,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingE
|
|||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeMusicTrendingLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
|
||||
|
|
@ -158,6 +160,16 @@ public class YoutubeService extends StreamingService {
|
|||
|
||||
// add kiosks here e.g.:
|
||||
try {
|
||||
list.addKioskEntry(
|
||||
(streamingService, url, id) -> new YoutubeMusicTrendingExtractor(
|
||||
YoutubeService.this,
|
||||
new YoutubeMusicTrendingLinkHandlerFactory().fromUrl(url),
|
||||
id
|
||||
),
|
||||
new YoutubeMusicTrendingLinkHandlerFactory(),
|
||||
"Trending Music"
|
||||
);
|
||||
|
||||
list.addKioskEntry(
|
||||
(streamingService, url, id) -> new YoutubeTrendingExtractor(
|
||||
YoutubeService.this,
|
||||
|
|
@ -173,6 +185,7 @@ public class YoutubeService extends StreamingService {
|
|||
}
|
||||
|
||||
return list;
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -0,0 +1,158 @@
|
|||
/*
|
||||
* Created by Christian Schabesberger on 12.08.17.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
|
||||
* YoutubeTrendingExtractor.java is part of NewPipe Extractor.
|
||||
*
|
||||
* NewPipe Extractor is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe Extractor is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
public class YoutubeMusicTrendingExtractor extends KioskExtractor<StreamInfoItem> {
|
||||
private JsonObject initialData;
|
||||
|
||||
public YoutubeMusicTrendingExtractor(final StreamingService service,
|
||||
final ListLinkHandler linkHandler,
|
||||
final String kioskId) {
|
||||
super(service, linkHandler, kioskId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFetchPage(@Nonnull final Downloader downloader)
|
||||
throws IOException, ExtractionException {
|
||||
// @formatter:off
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
|
||||
getExtractorContentCountry())
|
||||
.value("browseId", "MUtrending")
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
// @formatter:on
|
||||
|
||||
initialData = getJsonPostResponse("browse", body, getExtractorLocalization());
|
||||
}
|
||||
|
||||
@Override
|
||||
public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
|
||||
return InfoItemsPage.emptyPage();
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getName() throws ParsingException {
|
||||
final JsonObject header = initialData.getObject("header");
|
||||
String name = null;
|
||||
if (header.has("feedTabbedHeaderRenderer")) {
|
||||
name = getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), "title");
|
||||
} else if (header.has("c4TabbedHeaderRenderer")) {
|
||||
name = getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), "title");
|
||||
}
|
||||
|
||||
if (isNullOrEmpty(name)) {
|
||||
throw new ParsingException("Could not get Trending name");
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
||||
final JsonObject tabContent = getTrendingTabContent();
|
||||
|
||||
if (tabContent.has("richGridRenderer")) {
|
||||
tabContent.getObject("richGridRenderer")
|
||||
.getArray("contents")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
// Filter Trending shorts and Recently trending sections
|
||||
.filter(content -> content.has("richItemRenderer"))
|
||||
// .filter(shelfRenderer -> !shelfRenderer.has("music"))
|
||||
.map(content -> content.getObject("richItemRenderer")
|
||||
.getObject("content")
|
||||
.getObject("videoRenderer"))
|
||||
.forEachOrdered(videoRenderer -> collector.commit(
|
||||
new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser)));
|
||||
} else if (tabContent.has("sectionListRenderer")) {
|
||||
tabContent.getObject("sectionListRenderer")
|
||||
.getArray("contents")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.flatMap(content -> content.getObject("itemSectionRenderer")
|
||||
.getArray("contents")
|
||||
.stream())
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.map(content -> content.getObject("shelfRenderer"))
|
||||
// Filter Trending shorts and Recently trending sections which have a title,
|
||||
// contrary to normal trends
|
||||
.filter(shelfRenderer -> !shelfRenderer.has("title"))
|
||||
// .filter(shelfRenderer -> !shelfRenderer.has("music"))
|
||||
.flatMap(shelfRenderer -> shelfRenderer.getObject("content")
|
||||
.getObject("expandedShelfContentsRenderer")
|
||||
.getArray("items")
|
||||
.stream())
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.map(item -> item.getObject("videoRenderer"))
|
||||
.forEachOrdered(videoRenderer -> collector.commit(
|
||||
new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser)));
|
||||
}
|
||||
|
||||
return new InfoItemsPage<>(collector, null);
|
||||
}
|
||||
|
||||
private JsonObject getTrendingTabContent() throws ParsingException {
|
||||
return initialData.getObject("contents")
|
||||
.getObject("twoColumnBrowseResultsRenderer")
|
||||
.getArray("tabs")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.map(tab -> tab.getObject("tabRenderer"))
|
||||
.filter(tabRenderer -> tabRenderer.getBoolean("selected"))
|
||||
.filter(tabRenderer -> tabRenderer.has("content"))
|
||||
// There should be at most one tab selected
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new ParsingException("Could not get \"Now\" trending tab"))
|
||||
.getObject("content");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.List;
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
public final class YoutubeMusicTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
|
||||
|
||||
public String getUrl(final String id,
|
||||
final List<String> contentFilters,
|
||||
final String sortFilter) {
|
||||
return "https://www.youtube.com/feed/trending/music";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId(final String url) {
|
||||
return "Trending";
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean onAcceptUrl(final String url) {
|
||||
final URL urlObj;
|
||||
try {
|
||||
urlObj = Utils.stringToURL(url);
|
||||
} catch (final MalformedURLException e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
final String urlPath = urlObj.getPath();
|
||||
return Utils.isHTTP(urlObj) && (isYoutubeURL(urlObj) || isInvidioURL(urlObj))
|
||||
&& urlPath.equals("/feed/trending/music");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 12.08.17.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
|
||||
* YoutubeTrendingKioskInfoTest.java is part of NewPipe.
|
||||
*
|
||||
* NewPipe is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.schabi.newpipe.downloader.DownloaderFactory;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskInfo;
|
||||
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
|
||||
|
||||
/**
|
||||
* Test for {@link KioskInfo}
|
||||
*/
|
||||
public class YoutubeMusicTrendingKioskInfoTest {
|
||||
|
||||
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "kiosk";
|
||||
|
||||
static KioskInfo kioskInfo;
|
||||
|
||||
@BeforeAll
|
||||
public static void setUp()
|
||||
throws Exception {
|
||||
YoutubeTestsUtils.ensureStateless();
|
||||
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH));
|
||||
LinkHandlerFactory LinkHandlerFactory = ((StreamingService) YouTube).getKioskList().getListLinkHandlerFactoryByType("Trending");
|
||||
|
||||
kioskInfo = KioskInfo.getInfo(YouTube, LinkHandlerFactory.fromId("Trending").getUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getStreams() {
|
||||
assertFalse(kioskInfo.getRelatedItems().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getId() {
|
||||
assertTrue(kioskInfo.getId().equals("Trending Music")
|
||||
|| kioskInfo.getId().equals("Trends"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getName() {
|
||||
assertFalse(kioskInfo.getName().isEmpty());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 12.08.17.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
|
||||
* YoutubeTrendingLinkHandlerFactoryTest.java is part of NewPipe.
|
||||
*
|
||||
* NewPipe is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.schabi.newpipe.downloader.DownloaderTestImpl;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory;
|
||||
|
||||
/**
|
||||
* Test for {@link YoutubeTrendingLinkHandlerFactory}
|
||||
*/
|
||||
public class YoutubeMusicTrendingLinkHandlerFactoryTest {
|
||||
private static LinkHandlerFactory LinkHandlerFactory;
|
||||
|
||||
@BeforeAll
|
||||
public static void setUp() throws Exception {
|
||||
LinkHandlerFactory = YouTube.getKioskList().getListLinkHandlerFactoryByType("Trending Music");
|
||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getUrl()
|
||||
throws Exception {
|
||||
assertEquals(LinkHandlerFactory.fromId("").getUrl(), "https://www.youtube.com/feed/trending/music");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getId()
|
||||
throws Exception {
|
||||
assertEquals(LinkHandlerFactory.fromUrl("https://www.youtube.com/feed/trending/music").getId(), "Trending");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void acceptUrl() throws ParsingException {
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/trending/music"));
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/trending/music/?adsf=fjaj#fhe"));
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("http://www.youtube.com/feed/trending/music/"));
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("www.youtube.com/feed/trending/music"));
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("youtube.com/feed/trending/music"));
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("youtube.com/feed/trending/music/?akdsakjf=dfije&kfj=dkjak"));
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("https://youtube.com/feed/trending/music"));
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("m.youtube.com/feed/trending/music"));
|
||||
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("https://www.invidio.us/feed/trending/music"));
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("https://invidio.us/feed/trending/music"));
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("invidio.us/feed/trending/music"));
|
||||
|
||||
assertFalse(LinkHandlerFactory.acceptUrl("https://youtu.be/feed/trending/music"));
|
||||
assertFalse(LinkHandlerFactory.acceptUrl("kdskjfiiejfia"));
|
||||
assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/bullshit/feed/trending/music"));
|
||||
assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/trending/music/bullshit"));
|
||||
assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/bullshit/trending/music"));
|
||||
assertFalse(LinkHandlerFactory.acceptUrl("peter klaut aepferl youtube.com/feed/trending/music"));
|
||||
assertFalse(LinkHandlerFactory.acceptUrl("youtube.com/feed/trending/music askjkf"));
|
||||
assertFalse(LinkHandlerFactory.acceptUrl("askdjfi youtube.com/feed/trending/music askjkf"));
|
||||
assertFalse(LinkHandlerFactory.acceptUrl(" youtube.com/feed/trending/music"));
|
||||
assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/trending/music.html"));
|
||||
assertFalse(LinkHandlerFactory.acceptUrl(""));
|
||||
}
|
||||
}
|
||||
Ładowanie…
Reference in New Issue