diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java index 2a4ea6a9f..89ef3943b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java @@ -8,15 +8,12 @@ import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.Localization; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; -import java.time.OffsetDateTime; -import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -33,15 +30,6 @@ public final class MediaCCCParsingHelper { private MediaCCCParsingHelper() { } - public static OffsetDateTime parseDateFrom(final String textualUploadDate) - throws ParsingException { - try { - return OffsetDateTime.parse(textualUploadDate); - } catch (final DateTimeParseException e) { - throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); - } - } - /** * Check whether an id is a live stream id * @param id the {@code id} to check diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 99ddf5e08..23ebf4433 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getImageListFromLogoImageUrl; import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getThumbnailsFromStreamItem; -import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.parseDateFrom; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; @@ -27,6 +26,7 @@ import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.ExtractorHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.LocaleCompat; @@ -55,7 +55,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(parseDateFrom(getTextualUploadDate())); + return ExtractorHelper.parseDateWrapper(getTextualUploadDate()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java index ec9d00f3a..82687c328 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java @@ -4,9 +4,9 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.ExtractorHelper; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -66,10 +66,8 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor @Override public DateWrapper getUploadDate() throws ParsingException { final String date = getTextualUploadDate(); - if (date == null) { - return null; // event is in the future... - } - return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(date)); + // if null, event is in the future... + return date == null ? null : ExtractorHelper.parseDateWrapper(date); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java index d917eb2d7..74dc19b91 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java @@ -7,11 +7,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.ExtractorHelper; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.time.OffsetDateTime; -import java.time.format.DateTimeParseException; import java.util.List; public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { @@ -69,12 +68,8 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - try { - return new DateWrapper(OffsetDateTime.parse(getTextualUploadDate())); - } catch (final DateTimeParseException e) { - throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", - e); - } + final String date = getTextualUploadDate(); + return date == null ? null : ExtractorHelper.parseDateWrapper(date); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index e6e532034..01a088317 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -20,7 +20,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import static org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager.getTimeAgoParserFor; import static org.schabi.newpipe.extractor.services.youtube.ItagItem.APPROX_DURATION_MS_UNKNOWN; import static org.schabi.newpipe.extractor.services.youtube.ItagItem.CONTENT_LENGTH_UNKNOWN; import static org.schabi.newpipe.extractor.services.youtube.YoutubeDescriptionHelper.attributedDescriptionToHtml; @@ -60,6 +59,7 @@ import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider; import org.schabi.newpipe.extractor.services.youtube.PoTokenResult; @@ -104,6 +104,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; public class YoutubeStreamExtractor extends StreamExtractor { + private static final String PREMIERED = "Premiered "; + private static final String PREMIERED_ON = "Premiered on "; @Nullable private static PoTokenProvider poTokenProvider; @@ -172,19 +174,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nullable @Override public String getTextualUploadDate() throws ParsingException { - final var uploadDate = getUploadDate(); - if (uploadDate == null) { - return null; - } - return LocalDate.ofInstant(uploadDate.getInstant(), ZoneId.systemDefault()).toString(); - } - - @Override - public DateWrapper getUploadDate() throws ParsingException { final String dateStr = playerMicroFormatRenderer.getString("uploadDate", playerMicroFormatRenderer.getString("publishDate", "")); if (!dateStr.isEmpty()) { - return new DateWrapper(OffsetDateTime.parse(dateStr)); + return dateStr; } final var liveDetails = playerMicroFormatRenderer.getObject("liveBroadcastDetails"); @@ -192,50 +185,60 @@ public class YoutubeStreamExtractor extends StreamExtractor { liveDetails.getString("startTimestamp", "")); // a running live stream if (!timestamp.isEmpty()) { - return new DateWrapper(OffsetDateTime.parse(timestamp)); + return timestamp; } else if (getStreamType() == StreamType.LIVE_STREAM) { // this should never be reached, but a live stream without upload date is valid return null; } final var textObject = getVideoPrimaryInfoRenderer().getObject("dateText"); - return Optional.ofNullable(getTextFromObject(textObject)) - .flatMap(rendererDateText -> { - final Optional dateOptional; - - if (rendererDateText.startsWith("Premiered")) { - final String time = rendererDateText.substring(13); - - try { // Premiered 20 hours ago - final var localization = new Localization("en"); - return Optional.of(getTimeAgoParserFor(localization).parse(time)); - } catch (final Exception e) { - } - - // Premiered Feb 21, 2020 - dateOptional = parseOptionalDate(time, "MMM dd, yyyy") - // Premiered on 21 Feb 2020 - .or(() -> parseOptionalDate(time, "dd MMM yyyy")); - } else { - // Premiered on 21 Feb 2020 - dateOptional = parseOptionalDate(rendererDateText, "dd MMM yyyy"); - } - - return dateOptional.map(date -> { - final var instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant(); - return new DateWrapper(instant, true); - }); - }) - .orElseThrow(() -> new ParsingException("Could not get upload date")); + final String rendererDateText = getTextFromObject(textObject); + if (rendererDateText == null) { + return null; + } else if (rendererDateText.startsWith(PREMIERED_ON)) { // Premiered on 21 Feb 2020 + return rendererDateText.substring(PREMIERED_ON.length()); + } else if (rendererDateText.startsWith(PREMIERED)) { + // Premiered 20 hours ago / Premiered Feb 21, 2020 + return rendererDateText.substring(PREMIERED.length()); + } else { + return rendererDateText; + } } - private Optional parseOptionalDate(String date, String pattern) { + @Override + public DateWrapper getUploadDate() throws ParsingException { + final String dateText = getTextualUploadDate(); + if (dateText == null) { + return null; + } + try { - // TODO: this parses English formatted dates only, we need a better approach to - // parse the textual date + return new DateWrapper(OffsetDateTime.parse(dateText)); + } catch (final DateTimeParseException e) { + } + + try { // Premiered 20 hours ago + final var localization = new Localization("en"); + return TimeAgoPatternsManager.getTimeAgoParserFor(localization).parse(dateText); + } catch (final ParsingException e) { + } + + return parseOptionalDate(dateText, "MMM dd, yyyy") + .or(() -> parseOptionalDate(dateText.substring(3), "dd MMM yyyy")) + .map(date -> { + final var instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant(); + return new DateWrapper(instant, true); + }) + .orElseThrow(() -> new ParsingException("Could not parse upload date")); + } + + private Optional parseOptionalDate(final String date, final String pattern) { + try { + // TODO: this parses English formatted dates only, we need a better approach to parse + // the textual date final var formatter = DateTimeFormatter.ofPattern(pattern, Locale.ENGLISH); return Optional.of(LocalDate.parse(date, formatter)); - } catch (DateTimeParseException e) { + } catch (final DateTimeParseException e) { return Optional.empty(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index f75a97218..0d4b6e5d7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -246,7 +246,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Nullable @Override public String getTextualUploadDate() throws ParsingException { - if (getStreamType().equals(StreamType.LIVE_STREAM)) { + if (getStreamType() == StreamType.LIVE_STREAM) { return null; } @@ -274,7 +274,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - if (getStreamType().equals(StreamType.LIVE_STREAM)) { + if (getStreamType() == StreamType.LIVE_STREAM) { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 170291740..27f97e2ca 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -39,6 +39,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; +import java.time.Instant; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -71,7 +72,7 @@ public abstract class StreamExtractor extends Extractor { } /** - * A more general {@code Calendar} instance set to the date provided by the service.
+ * A more general {@link Instant} instance set to the date provided by the service.
* Implementations usually will just parse the date returned from the {@link * #getTextualUploadDate()}. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java index 12023ca4c..0619b52e5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java @@ -5,16 +5,31 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfo; +import java.time.OffsetDateTime; +import java.time.format.DateTimeParseException; import java.util.Collections; import java.util.List; +import javax.annotation.Nonnull; + public final class ExtractorHelper { private ExtractorHelper() { } + @Nonnull + public static DateWrapper parseDateWrapper(@Nonnull final String date) throws ParsingException { + try { + return new DateWrapper(OffsetDateTime.parse(date)); + } catch (final DateTimeParseException e) { + throw new ParsingException("Could not parse date: \"" + date + "\"", e); + } + } + public static InfoItemsPage getItemsPageOrLogError( final Info info, final ListExtractor extractor) { try {