diff --git a/build.gradle b/build.gradle index e6dca3626..23415b1e0 100644 --- a/build.gradle +++ b/build.gradle @@ -31,6 +31,7 @@ allprojects { jsr305Version = "3.0.2" junitVersion = "5.13.3" checkstyleVersion = "10.4" + immutablesVersion = "2.10.1" } } diff --git a/extractor/build.gradle b/extractor/build.gradle index 7baad75d3..5310ef013 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -52,6 +52,12 @@ dependencies { testImplementation "com.squareup.okhttp3:okhttp:4.12.0" testImplementation 'com.google.code.gson:gson:2.13.1' + testImplementation "org.immutables:value:$immutablesVersion" + testAnnotationProcessor "org.immutables:value:$immutablesVersion" + +} +repositories { + mavenCentral() } protobuf { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java index ffc29a61c..2f0a2bdd2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java @@ -69,4 +69,12 @@ public class DateWrapper implements Serializable { public boolean isApproximation() { return isApproximation; } + + @Override + public String toString() { + return "DateWrapper{" + + "offsetDateTime=" + offsetDateTime + + ", isApproximation=" + isApproximation + + '}'; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 67cd533e9..595862bde 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -121,7 +121,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public long getTimeStamp() throws ParsingException { - return getTimestampSeconds("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); + final var timestamp = getTimestampSeconds("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); + return timestamp == -2 ? 0 : timestamp; } @Override @@ -170,7 +171,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { try { final JsonArray transcodings = track.getObject("media") - .getArray("transcodings"); + .getArray("transcodings"); if (!isNullOrEmpty(transcodings)) { // Get information about what stream formats are available extractAudioStreams(transcodings, audioStreams); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java index e7809c52a..421022ef0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud.linkHandler; +import java.util.regex.Pattern; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; @@ -9,11 +11,18 @@ import org.schabi.newpipe.extractor.utils.Utils; public final class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory { private static final SoundcloudStreamLinkHandlerFactory INSTANCE = new SoundcloudStreamLinkHandlerFactory(); - private static final String URL_PATTERN = "^https?://(www\\.|m\\.|on\\.)?" - + "soundcloud.com/[0-9a-z_-]+" - + "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$"; - private static final String API_URL_PATTERN = "^https?://api-v2\\.soundcloud.com" - + "/(tracks|albums|sets|reposts|followers|following)/([0-9a-z_-]+)/"; + + private static final Pattern URL_PATTERN = Pattern.compile( + "^https?://(?:www\\.|m\\.|on\\.)?" + + "soundcloud.com/[0-9a-z_-]+" + + "/(?!(?:tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?(?:[#?].*)?$" + ); + + private static final Pattern API_URL_PATTERN = Pattern.compile( + "^https?://api-v2\\.soundcloud.com" + + "/(tracks|albums|sets|reposts|followers|following)/([0-9a-z_-]+)/" + ); + private SoundcloudStreamLinkHandlerFactory() { } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index cb28c5e6f..0cf4654e6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -44,50 +44,113 @@ public final class Parser { } } + @Nonnull + public static Matcher matchOrThrow(@Nonnull final Pattern pattern, + final String input) throws RegexException { + final Matcher matcher = pattern.matcher(input); + if (matcher.find()) { + return matcher; + } else { + String errorMessage = "Failed to find pattern \"" + pattern.pattern() + "\""; + if (input.length() <= 1024) { + errorMessage += " inside of \"" + input + "\""; + } + throw new RegexException(errorMessage); + } + } + + /** + * Matches group 1 of the given pattern against the input + * and returns the matched group + * + * @param pattern The regex pattern to match. + * @param input The input string to match against. + * @return The matching group as a string. + * @throws RegexException If the pattern does not match the input or if the group is not found. + */ + @Nonnull public static String matchGroup1(final String pattern, final String input) throws RegexException { return matchGroup(pattern, input, 1); } - public static String matchGroup1(final Pattern pattern, - final String input) throws RegexException { + /** + * Matches group 1 of the given pattern against the input + * and returns the matched group + * + * @param pattern The regex pattern to match. + * @param input The input string to match against. + * @return The matching group as a string. + * @throws RegexException If the pattern does not match the input or if the group is not found. + */ + @Nonnull + public static String matchGroup1(final Pattern pattern, final String input) + throws RegexException { return matchGroup(pattern, input, 1); } - public static String matchGroup(final String pattern, - final String input, - final int group) throws RegexException { + /** + * Matches the specified group of the given pattern against the input, + * and returns the matched group + * + * @param pattern The regex pattern to match. + * @param input The input string to match against. + * @param group The group number to retrieve (1-based index). + * @return The matching group as a string. + * @throws RegexException If the pattern does not match the input or if the group is not found. + */ + @Nonnull + public static String matchGroup(final String pattern, final String input, final int group) + throws RegexException { return matchGroup(Pattern.compile(pattern), input, group); } - public static String matchGroup(@Nonnull final Pattern pat, + /** + * Matches the specified group of the given pattern against the input, + * and returns the matched group + * + * @param pattern The regex pattern to match. + * @param input The input string to match against. + * @param group The group number to retrieve (1-based index). + * @return The matching group as a string. + * @throws RegexException If the pattern does not match the input or if the group is not found. + */ + @Nonnull + public static String matchGroup(@Nonnull final Pattern pattern, final String input, - final int group) throws RegexException { - final Matcher matcher = pat.matcher(input); - final boolean foundMatch = matcher.find(); - if (foundMatch) { - return matcher.group(group); - } else { - // only pass input to exception message when it is not too long - if (input.length() > 1024) { - throw new RegexException("Failed to find pattern \"" + pat.pattern() + "\""); - } else { - throw new RegexException("Failed to find pattern \"" + pat.pattern() - + "\" inside of \"" + input + "\""); - } - } + final int group) + throws RegexException { + return matchOrThrow(pattern, input).group(group); } + /** + * Matches multiple patterns against the input string and + * returns the first successful matcher + * + * @param patterns The array of regex patterns to match. + * @param input The input string to match against. + * @return A {@code Matcher} for the first successful match. + * @throws RegexException If no patterns match the input or if {@code patterns} is empty. + */ public static String matchGroup1MultiplePatterns(final Pattern[] patterns, final String input) throws RegexException { return matchMultiplePatterns(patterns, input).group(1); } + /** + * Matches multiple patterns against the input string and + * returns the first successful matcher + * + * @param patterns The array of regex patterns to match. + * @param input The input string to match against. + * @return A {@code Matcher} for the first successful match. + * @throws RegexException If no patterns match the input or if {@code patterns} is empty. + */ public static Matcher matchMultiplePatterns(final Pattern[] patterns, final String input) throws RegexException { - Parser.RegexException exception = null; - for (final Pattern pattern : patterns) { - final Matcher matcher = pattern.matcher(input); + RegexException exception = null; + for (final var pattern : patterns) { + final var matcher = pattern.matcher(input); if (matcher.find()) { return matcher; } else if (exception == null) { @@ -110,14 +173,11 @@ public final class Parser { } public static boolean isMatch(final String pattern, final String input) { - final Pattern pat = Pattern.compile(pattern); - final Matcher mat = pat.matcher(input); - return mat.find(); + return isMatch(Pattern.compile(pattern), input); } public static boolean isMatch(@Nonnull final Pattern pattern, final String input) { - final Matcher mat = pattern.matcher(input); - return mat.find(); + return pattern.matcher(input).find(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index c061ce30f..38eb5b33f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -110,6 +110,16 @@ public final class Utils { * @param url the url to be tested */ public static void checkUrl(final String pattern, final String url) throws ParsingException { + checkUrl(Pattern.compile(pattern), url); + } + + /** + * Check if the url matches the pattern. + * + * @param pattern the pattern that will be used to check the url + * @param url the url to be tested + */ + public static void checkUrl(final Pattern pattern, final String url) throws ParsingException { if (isNullOrEmpty(url)) { throw new IllegalArgumentException("Url can't be null or empty"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java index e5bed1d69..7b2f06c43 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java @@ -8,6 +8,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -168,6 +169,13 @@ public class ExtractorAsserts { "'" + shouldBeContained + "' should be contained inside '" + container + "'"); } + public static void assertMatches(final Pattern pattern, final String input) { + assertNotNull(pattern, "pattern is null"); + assertNotNull(input, "input is null"); + assertTrue(pattern.matcher(input).find(), + "Pattern '" + pattern + "' not found in input '" + input + "'"); + } + public static void assertTabsContain(@Nonnull final List tabs, @Nonnull final String... expectedTabs) { final Set tabSet = tabs.stream() diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/ImmutableStyle.java b/extractor/src/test/java/org/schabi/newpipe/extractor/ImmutableStyle.java new file mode 100644 index 000000000..c69ea5aff --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/ImmutableStyle.java @@ -0,0 +1,30 @@ +package org.schabi.newpipe.extractor; + +import org.immutables.value.Value; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Target; + +// CHECKSTYLE:OFF +/** + * Custom style for generated Immutables. + * See Style. + *

+ * - Abstract types start with 'I' (e.g., IExample).

+ * - Concrete immutable types do not have a prefix (e.g., Example).

+ * - Getters are prefixed with 'get', 'is', or no prefix.

+ * - Strict builder pattern is enforced.

+ */ +// CHECKSTYLE:ON +@Target({ElementType.PACKAGE, ElementType.TYPE}) +@Value.Style( + get = {"get*", "is*", "*"}, // Methods matching these prefixes will be used as getters. + // Methods matching these patterns can NOT be used as setters. + typeAbstract = {"I*"}, // Abstract types start with I + typeImmutable = "*", // Generated concrete Immutable types will not have the I prefix + visibility = Value.Style.ImplementationVisibility.PUBLIC, + strictBuilder = true, + defaultAsDefault = true, // https://immutables.github.io/immutable.html#default-attributes + jdkOnly = true +) +public @interface ImmutableStyle { } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultSoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultSoundcloudStreamExtractorTest.java new file mode 100644 index 000000000..5cdf7d933 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultSoundcloudStreamExtractorTest.java @@ -0,0 +1,57 @@ +package org.schabi.newpipe.extractor.services; + +import org.junit.jupiter.api.Test; +import org.schabi.newpipe.extractor.ExtractorAsserts; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.services.testcases.SoundcloudStreamExtractorTestCase; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.DeliveryMethod; + +import java.util.List; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.*; + +public abstract class ParameterisedDefaultSoundcloudStreamExtractorTest + extends ParameterisedDefaultStreamExtractorTest { + protected ParameterisedDefaultSoundcloudStreamExtractorTest(SoundcloudStreamExtractorTestCase testCase) { + super(testCase); + } + + final Pattern mp3CdnUrlPattern = Pattern.compile("-media\\.sndcdn\\.com/[a-zA-Z0-9]{12}\\.128\\.mp3"); + + @Override + @Test + public void testAudioStreams() throws Exception { + super.testAudioStreams(); + final List audioStreams = extractor.getAudioStreams(); + assertEquals(3, audioStreams.size()); // 2 MP3 streams (1 progressive, 1 HLS) and 1 OPUS + audioStreams.forEach(audioStream -> { + final DeliveryMethod deliveryMethod = audioStream.getDeliveryMethod(); + final String mediaUrl = audioStream.getContent(); + if (audioStream.getFormat() == MediaFormat.OPUS) { + assertSame(DeliveryMethod.HLS, deliveryMethod, + "Wrong delivery method for stream " + audioStream.getId() + ": " + + deliveryMethod); + // Assert it's an OPUS 64 kbps media playlist URL which comes from an HLS + // SoundCloud CDN + ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl); + ExtractorAsserts.assertContains(".64.opus", mediaUrl); + } else if (audioStream.getFormat() == MediaFormat.MP3) { + if (deliveryMethod == DeliveryMethod.PROGRESSIVE_HTTP) { + // Assert it's a MP3 128 kbps media URL which comes from a progressive + // SoundCloud CDN + ExtractorAsserts.assertMatches(mp3CdnUrlPattern, mediaUrl); + } else if (deliveryMethod == DeliveryMethod.HLS) { + // Assert it's a MP3 128 kbps media HLS playlist URL which comes from an HLS + // SoundCloud CDN + ExtractorAsserts.assertContains("-hls-media.sndcdn.com", mediaUrl); + ExtractorAsserts.assertContains(".128.mp3", mediaUrl); + } else { + fail("Wrong delivery method for stream " + audioStream.getId() + ": " + + deliveryMethod); + } + } + }); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultStreamExtractorTest.java new file mode 100644 index 000000000..4a84ca666 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultStreamExtractorTest.java @@ -0,0 +1,88 @@ +package org.schabi.newpipe.extractor.services; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestInstance; +import org.schabi.newpipe.downloader.DownloaderTestImpl; +import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.services.testcases.DefaultStreamExtractorTestCase; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Locale; + +/** + * Test for {@link StreamExtractor} + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class ParameterisedDefaultStreamExtractorTest extends DefaultStreamExtractorTest { + protected TTestCase testCase; + protected StreamExtractor extractor; + + protected ParameterisedDefaultStreamExtractorTest(TTestCase testCase) + { + this.testCase = testCase; + } + + @BeforeAll + public void setUp() throws Exception { + if (extractor != null) { + throw new IllegalStateException("extractor already initialized before BeforeAll"); + } + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = testCase.service().getStreamExtractor(testCase.url()); + extractor.fetchPage(); + } + + /// + /// DefaultExtractorTest overrides + /// + + @Override public StreamExtractor extractor() throws Exception { return extractor; } + + @Override public StreamingService expectedService() throws Exception { return testCase.service(); } + @Override public String expectedName() throws Exception { return testCase.name(); } + @Override public String expectedId() throws Exception { return testCase.id(); } + @Override public String expectedUrlContains() throws Exception { return testCase.urlContains(); } + @Override public String expectedOriginalUrlContains() throws Exception { return testCase.originalUrlContains(); } + + /// + /// DefaultStreamExtractorTest overrides + /// + @Override public StreamType expectedStreamType() { return testCase.streamType(); } + @Override public String expectedUploaderName() { return testCase.uploaderName(); } + @Override public String expectedUploaderUrl() { return testCase.uploaderUrl(); } + @Override public boolean expectedUploaderVerified() { return testCase.uploaderVerified(); } + @Override public long expectedUploaderSubscriberCountAtLeast() { return testCase.uploaderSubscriberCountAtLeast(); } + @Override public String expectedSubChannelName() { return testCase.subChannelName(); } + @Override public String expectedSubChannelUrl() { return testCase.subChannelUrl(); } + @Override public boolean expectedDescriptionIsEmpty() { return testCase.descriptionIsEmpty(); } + @Override public List expectedDescriptionContains() { return testCase.descriptionContains(); } + @Override public long expectedLength() { return testCase.length(); } + @Override public long expectedTimestamp() { return testCase.timestamp(); } + @Override public long expectedViewCountAtLeast() { return testCase.viewCountAtLeast(); } + @Override @Nullable public String expectedUploadDate() { return testCase.uploadDate(); } + @Override @Nullable public String expectedTextualUploadDate() { return testCase.textualUploadDate(); } + @Override public long expectedLikeCountAtLeast() { return testCase.likeCountAtLeast(); } + @Override public long expectedDislikeCountAtLeast() { return testCase.dislikeCountAtLeast(); } + @Override public boolean expectedHasRelatedItems() { return testCase.hasRelatedItems(); } + @Override public int expectedAgeLimit() { return testCase.ageLimit(); } + @Override @Nullable public String expectedErrorMessage() { return testCase.errorMessage(); } + @Override public boolean expectedHasVideoStreams() { return testCase.hasVideoStreams(); } + @Override public boolean expectedHasAudioStreams() { return testCase.hasAudioStreams(); } + @Override public boolean expectedHasSubtitles() { return testCase.hasSubtitles(); } + @Override @Nullable public String expectedDashMpdUrlContains() { return testCase.dashMpdUrlContains(); } + @Override public boolean expectedHasFrames() { return testCase.hasFrames(); } + @Override public String expectedHost() { return testCase.host(); } + @Override public StreamExtractor.Privacy expectedPrivacy() { return testCase.privacy(); } + @Override public String expectedCategory() { return testCase.category(); } + @Override public String expectedLicence() { return testCase.licence(); } + @Override public Locale expectedLanguageInfo() { return testCase.languageInfo(); } + @Override public List expectedTags() { return testCase.tags(); } + @Override public String expectedSupportInfo() { return testCase.supportInfo(); } + @Override public int expectedStreamSegmentsCount() { return testCase.streamSegmentsCount(); } + @Override public List expectedMetaInfo() { return testCase.metaInfo(); } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java index a4e81dd9c..8405b8fd0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -2,43 +2,44 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.schabi.newpipe.downloader.DownloaderTestImpl; -import org.schabi.newpipe.extractor.ExtractorAsserts; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException; import org.schabi.newpipe.extractor.exceptions.SoundCloudGoPlusContentException; import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; +import org.schabi.newpipe.extractor.services.ParameterisedDefaultSoundcloudStreamExtractorTest; +import org.schabi.newpipe.extractor.services.testcases.SoundcloudStreamExtractorTestCase; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamType; -import java.util.Arrays; import java.util.Collections; import java.util.List; import javax.annotation.Nullable; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.fail; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; public class SoundcloudStreamExtractorTest { private static final String SOUNDCLOUD = "https://soundcloud.com/"; - public static class SoundcloudGeoRestrictedTrack extends DefaultStreamExtractorTest { + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class SoundcloudGeoRestrictedTrack extends DefaultStreamExtractorTest { private static final String ID = "one-touch"; private static final String UPLOADER = SOUNDCLOUD + "jessglynne"; private static final int TIMESTAMP = 0; private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP; - private static StreamExtractor extractor; + private StreamExtractor extractor; @BeforeAll - public static void setUp() throws Exception { + public void setUp() throws Exception { + if (extractor != null) { + throw new IllegalStateException("extractor already initialized before BeforeAll"); + } NewPipe.init(DownloaderTestImpl.getInstance()); extractor = SoundCloud.getStreamExtractor(URL); try { @@ -84,15 +85,20 @@ public class SoundcloudStreamExtractorTest { } } - public static class SoundcloudGoPlusTrack extends DefaultStreamExtractorTest { + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class SoundcloudGoPlusTrack extends DefaultStreamExtractorTest { private static final String ID = "places"; private static final String UPLOADER = SOUNDCLOUD + "martinsolveig"; private static final int TIMESTAMP = 0; private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP; - private static StreamExtractor extractor; + private StreamExtractor extractor; @BeforeAll - public static void setUp() throws Exception { + public void setUp() throws Exception { + if (extractor != null) { + throw new IllegalStateException("extractor already initialized before BeforeAll"); + } NewPipe.init(DownloaderTestImpl.getInstance()); extractor = SoundCloud.getStreamExtractor(URL); try { @@ -133,7 +139,6 @@ public class SoundcloudStreamExtractorTest { @Override public long expectedDislikeCountAtLeast() { return -1; } @Override public boolean expectedHasAudioStreams() { return false; } @Override public boolean expectedHasVideoStreams() { return false; } - @Override public boolean expectedHasRelatedItems() { return true; } @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } @Override public int expectedStreamSegmentsCount() { return 0; } @@ -141,82 +146,79 @@ public class SoundcloudStreamExtractorTest { @Override public String expectedCategory() { return "Dance"; } } - static class CreativeCommonsOpenMindsEp21 extends DefaultStreamExtractorTest { - private static final String ID = "open-minds-ep-21-dr-beth-harris-and-dr-steven-zucker-of-smarthistory"; - private static final String UPLOADER = SOUNDCLOUD + "wearecc"; - private static final int TIMESTAMP = 69; - private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP; - private static StreamExtractor extractor; - - @BeforeAll - static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = SoundCloud.getStreamExtractor(URL); - extractor.fetchPage(); + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class SoundcloudTrackTest1 extends ParameterisedDefaultSoundcloudStreamExtractorTest { + public SoundcloudTrackTest1() { + super( + SoundcloudStreamExtractorTestCase.builder() + .url("https://soundcloud.com/user-904087338/nether#t=45") + .id("2057071056") + .name("Nether") + .uploaderName("Ambient Ghost") + .uploadDate("2025-03-18 12:19:19.000") + .textualUploadDate("2025-03-18 12:19:19") + .length(145) + .licence("all-rights-reserved") + .descriptionIsEmpty(true) + .viewCountAtLeast(1029) + .likeCountAtLeast(12) + .build() + ); } + } - @Override public StreamExtractor extractor() { return extractor; } - @Override public StreamingService expectedService() { return SoundCloud; } - @Override public String expectedName() { return "Open Minds, Ep 21: Dr. Beth Harris and Dr. Steven Zucker of Smarthistory"; } - @Override public String expectedId() { return "1356023209"; } - @Override public String expectedUrlContains() { return UPLOADER + "/" + ID; } - @Override public String expectedOriginalUrlContains() { return URL; } + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class SoundcloudTrackTest2 extends ParameterisedDefaultSoundcloudStreamExtractorTest { + public SoundcloudTrackTest2() { + super( + SoundcloudStreamExtractorTestCase.builder() + .url("https://soundcloud.com/kaleidocollective/2subtact-splinter") + .id("230211123") + .name("Subtact - Splinter") + .uploaderVerified(true) + .uploaderName("Kaleido") + .uploadDate("2015-10-26 20:55:30.000") + .textualUploadDate("2015-10-26 20:55:30") + .length(225) + .licence("all-rights-reserved") + .descriptionIsEmpty(false) + .addDescriptionContains("follow @subtact", + "-twitter:", + "twitter.com/Subtact", + "-facebook:", + "www.facebook.com/subtact?fref=ts") + .viewCountAtLeast(157874) + .likeCountAtLeast(3142) + .category("ʕ•ᴥ•ʔ") + .build() + ); + } + } - @Override public StreamType expectedStreamType() { return StreamType.AUDIO_STREAM; } - @Override public String expectedUploaderName() { return "Creative Commons"; } - @Override public String expectedUploaderUrl() { return UPLOADER; } - @Override public List expectedDescriptionContains() { - return Arrays.asList("Smarthistory is a center for public art history", - "experts who want to share their knowledge with learners around the world", - "Available for use under the CC BY 3.0 license"); } - @Override public long expectedLength() { return 1500; } - @Override public long expectedTimestamp() { return TIMESTAMP; } - @Override public long expectedViewCountAtLeast() { return 15000; } - @Nullable @Override public String expectedUploadDate() { return "2022-10-03 18:49:49.000"; } - @Nullable @Override public String expectedTextualUploadDate() { return "2022-10-03 18:49:49"; } - @Override public long expectedLikeCountAtLeast() { return 10; } - @Override public long expectedDislikeCountAtLeast() { return -1; } - @Override public boolean expectedHasRelatedItems() { return false; } - @Override public boolean expectedHasVideoStreams() { return false; } - @Override public boolean expectedHasSubtitles() { return false; } - @Override public boolean expectedHasFrames() { return false; } - @Override public int expectedStreamSegmentsCount() { return 0; } - @Override public String expectedLicence() { return "cc-by"; } - - @Override - @Test - public void testAudioStreams() throws Exception { - super.testAudioStreams(); - final List audioStreams = extractor.getAudioStreams(); - assertEquals(3, audioStreams.size()); // 2 MP3 streams (1 progressive, 1 HLS) and 1 OPUS - audioStreams.forEach(audioStream -> { - final DeliveryMethod deliveryMethod = audioStream.getDeliveryMethod(); - final String mediaUrl = audioStream.getContent(); - if (audioStream.getFormat() == MediaFormat.OPUS) { - assertSame(DeliveryMethod.HLS, deliveryMethod, - "Wrong delivery method for stream " + audioStream.getId() + ": " - + deliveryMethod); - // Assert it's an OPUS 64 kbps media playlist URL which comes from an HLS - // SoundCloud CDN - ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl); - ExtractorAsserts.assertContains(".64.opus", mediaUrl); - } else if (audioStream.getFormat() == MediaFormat.MP3) { - if (deliveryMethod == DeliveryMethod.PROGRESSIVE_HTTP) { - // Assert it's a MP3 128 kbps media URL which comes from a progressive - // SoundCloud CDN - ExtractorAsserts.assertContains("-media.sndcdn.com/cyaz0oXJYbdt.128.mp3", - mediaUrl); - } else if (deliveryMethod == DeliveryMethod.HLS) { - // Assert it's a MP3 128 kbps media HLS playlist URL which comes from an HLS - // SoundCloud CDN - ExtractorAsserts.assertContains("-hls-media.sndcdn.com", mediaUrl); - ExtractorAsserts.assertContains(".128.mp3", mediaUrl); - } else { - fail("Wrong delivery method for stream " + audioStream.getId() + ": " - + deliveryMethod); - } - } - }); + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class SoundcloudTrackTest3 extends ParameterisedDefaultSoundcloudStreamExtractorTest { + public SoundcloudTrackTest3() { + super(SoundcloudStreamExtractorTestCase.builder() + .url("https://soundcloud.com/wearecc/open-minds-ep-21-dr-beth-harris-and-dr-steven-zucker-of-smarthistory") + .id("1356023209") + .name("Open Minds, Ep 21: Dr. Beth Harris and Dr. Steven Zucker of Smarthistory") + .uploaderName("Creative Commons") + .uploadDate("2022-10-03 18:49:49.000") + .textualUploadDate("2022-10-03 18:49:49") + .hasRelatedItems(false) + .length(1500) + .licence("cc-by") + .descriptionIsEmpty(false) + .addDescriptionContains("Smarthistory is a center for public art history", + "experts who want to share their knowledge with learners around the world", + "Available for use under the CC BY 3.0 license") + .viewCountAtLeast(15000) + .likeCountAtLeast(10) + .build() + ); } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultExtractorTestCase.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultExtractorTestCase.java new file mode 100644 index 000000000..f43656161 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultExtractorTestCase.java @@ -0,0 +1,17 @@ +package org.schabi.newpipe.extractor.services.testcases; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.services.DefaultExtractorTest; + +/** + * Test case base class for {@link DefaultExtractorTest} + */ +public interface DefaultExtractorTestCase { + public abstract StreamingService service(); + public abstract String name(); + public abstract String id(); + public abstract String url(); + public default String originalUrl() { return url(); } + public default String urlContains() { return url();} + public default String originalUrlContains() { return originalUrl(); } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultStreamExtractorTestCase.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultStreamExtractorTestCase.java new file mode 100644 index 000000000..4190b5ceb --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultStreamExtractorTestCase.java @@ -0,0 +1,90 @@ +package org.schabi.newpipe.extractor.services.testcases; + +import static org.schabi.newpipe.extractor.stream.StreamExtractor.UNKNOWN_SUBSCRIBER_COUNT; +import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; + +import javax.annotation.Nullable; + +/** + * Test case base class for {@link org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest}

+ * Ideally you will supply a regex matcher that the url that will automatically parse + * certain values for the tests.

+ * Ones that can't be derived from the url should be overridden in the test case. + */ +public interface DefaultStreamExtractorTestCase extends DefaultExtractorTestCase { + /** + * Returns matcher for the URL

+ * Implementations should throw IllegalArgumentException if the pattern does not match + */ + Matcher urlMatcher(); + + default String getGroupFromUrl(String groupName) { + return urlMatcher().group(groupName); + } + + default int getGroupEndIndexFromUrl(String groupName) { + return urlMatcher().end(groupName); + } + + default String id() { return getGroupFromUrl("id"); } + + default String uploader() { return getGroupFromUrl("uploader"); } + + StreamType streamType(); + String uploaderName(); + default String uploaderUrl() { + final int groupEndIndex = getGroupEndIndexFromUrl("uploader"); + if (groupEndIndex < 0) { + return ""; // no uploader group found in url + } + return url().substring(0, groupEndIndex); + } + default boolean uploaderVerified() { return false; } + default long uploaderSubscriberCountAtLeast() { return UNKNOWN_SUBSCRIBER_COUNT; } // default: unknown + default String subChannelName() { return ""; } // default: no subchannel + default String subChannelUrl() { return ""; } // default: no subchannel + default boolean descriptionIsEmpty() { return false; } // default: description is not empty + List descriptionContains(); + long length(); + default int timestamp() { return 0; } // default: there is no timestamp + long viewCountAtLeast(); + + @Nullable + String uploadDate(); // format: yyyy-MM-dd HH:mm:ss.SSS + @Nullable + String textualUploadDate(); + long likeCountAtLeast(); + long dislikeCountAtLeast(); + default boolean hasRelatedItems() { return true; } // default: there are related videos + default int ageLimit() { return StreamExtractor.NO_AGE_LIMIT; } // default: no limit + @Nullable + default String errorMessage() { return null; } // default: no error message + default boolean hasVideoStreams() { return true; } // default: there are video streams + default boolean hasAudioStreams() { return true; } // default: there are audio streams + default boolean hasSubtitles() { return true; } // default: there are subtitles streams + @Nullable + default String dashMpdUrlContains() { return null; } // default: no dash mpd + default boolean hasFrames() { return true; } // default: there are frames + @Nullable + default String host() { return ""; } // default: no host for centralized platforms + @Nullable + default StreamExtractor.Privacy privacy() { return StreamExtractor.Privacy.PUBLIC; } // default: public + default String category() { return ""; } // default: no category + default String licence() { return ""; } // default: no licence + @Nullable + default Locale languageInfo() { return null; } // default: no language info available + @Nullable + default List tags() { return Collections.emptyList(); } // default: no tags + @Nullable + default String supportInfo() { return ""; } // default: no support info available + default int streamSegmentsCount() { return -1; } // return 0 or greater to test (default is -1 to ignore) + @Nullable + default List metaInfo() { return Collections.emptyList(); } // default: no metadata info available +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/ISoundcloudStreamExtractorTestCase.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/ISoundcloudStreamExtractorTestCase.java new file mode 100644 index 000000000..7a477abfa --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/ISoundcloudStreamExtractorTestCase.java @@ -0,0 +1,110 @@ +package org.schabi.newpipe.extractor.services.testcases; + +import org.immutables.value.Value; +import org.schabi.newpipe.extractor.ImmutableStyle; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudStreamExtractorTest; + +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +// CHECKSTYLE:OFF +/** + * Immutable definition of {@link DefaultStreamExtractorTestCase} + * for {@link SoundcloudStreamExtractorTest} streams + * @see SoundcloudStreamExtractorTest + */ +// CHECKSTYLE:ON +@ImmutableStyle +@Value.Immutable +public interface ISoundcloudStreamExtractorTestCase extends DefaultStreamExtractorTestCase { + + /** + * Pattern for matching soundcloud stream URLs + * Matches URLs of the form: + * 

+     * https://soundcloud.com/user-904087338/nether#t=46
+     *
+ */ + Pattern URL_PATTERN = Pattern.compile( + "^https?://(?:www\\.|m\\.|on\\.)?soundcloud\\.com/" + + "(?[0-9a-z_-]+)/(?!(?:tracks|albums|sets|reposts|followers|following)/?$)" + + "(?[0-9a-z_-]+)/?" + + "([#?](t=(?\\d+)|.*))?$" + ); + + /** + * Returns the named group from the URL, or an empty string if not found. + */ + default String getGroupFromUrl(String group) { + try { + final String value = urlMatcher().group(group); + return value != null ? value : ""; + } catch (IllegalArgumentException | IllegalStateException e) { + return ""; + } + } + + /** + * Returns the end index of the named group from the URL, or -1 if not found. + */ + default int getGroupEndIndexFromUrl(String group) { + try { + return urlMatcher().end(group); + } catch (IllegalArgumentException | IllegalStateException e) { + return -1; + } + } + + default Matcher urlMatcher() { + try { + return Parser.matchOrThrow(URL_PATTERN, url()); + } catch (RegexException e) { + throw new IllegalArgumentException("URL does not match expected SoundCloud pattern: " + url(), e); + } + } + + default String urlContains() { + final int groupEndIndex = getGroupEndIndexFromUrl("id"); + if (groupEndIndex < 0) { + return url(); // no id group found in url + } + return url().substring(0, groupEndIndex); + } + + @Value.Derived + default StreamingService service() { return SoundCloud; } + + @Value.Derived + @Override + default StreamType streamType() { return StreamType.AUDIO_STREAM; } + + @Override + default int timestamp() { + try { + return Integer.parseInt(getGroupFromUrl("timestamp")); + } + catch (NumberFormatException e) { + // Return 0 if no timestamp + return 0; + } + } + + @Override + default long dislikeCountAtLeast() { return -1; } // default: soundcloud has no dislikes + + @Override + default boolean hasVideoStreams() { return false; } // default: soundcloud has no video streams + + @Override + default boolean hasSubtitles() { return false; } // default: soundcloud has no subtitles + + default boolean hasFrames() { return false; } // default: soundcloud has no frames + + default int streamSegmentsCount() { return 0; } +}