Unverified Commit 8cbdec67 authored by Tobias Groza's avatar Tobias Groza Committed by GitHub
Browse files

Merge pull request #417 from Isira-Seneviratne/Use_Java_8_Date_Time_API

Use the Java 8 Date/Time API.
parents 6cc50b57 fcdfe7d9
......@@ -13,6 +13,8 @@ If you're using Gradle, you could add NewPipe Extractor as a dependency with the
1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.1'`the `dependencies` in your `build.gradle`. Replace `v0.20.1` with the latest release.
**Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required.
### Testing changes
To test changes quickly you can build the library locally. A good approach would be to add something like the following to your `settings.gradle`:
......
......@@ -2,8 +2,8 @@ allprojects {
apply plugin: 'java-library'
apply plugin: 'maven'
sourceCompatibility = 1.7
targetCompatibility = 1.7
sourceCompatibility = 1.8
targetCompatibility = 1.8
version 'v0.20.2'
group 'com.github.TeamNewPipe'
......
......@@ -3,30 +3,61 @@ package org.schabi.newpipe.extractor.localization;
import edu.umd.cs.findbugs.annotations.NonNull;
import java.io.Serializable;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Calendar;
import java.util.GregorianCalendar;
/**
* A wrapper class that provides a field to describe if the date is precise or just an approximation.
* A wrapper class that provides a field to describe if the date/time is precise or just an approximation.
*/
public class DateWrapper implements Serializable {
@NonNull private final Calendar date;
@NonNull private final OffsetDateTime offsetDateTime;
private final boolean isApproximation;
public DateWrapper(@NonNull Calendar date) {
this(date, false);
/**
* @deprecated Use {@link #DateWrapper(OffsetDateTime)} instead.
*/
@Deprecated
public DateWrapper(@NonNull Calendar calendar) {
this(calendar, false);
}
/**
* @deprecated Use {@link #DateWrapper(OffsetDateTime, boolean)} instead.
*/
@Deprecated
public DateWrapper(@NonNull Calendar calendar, boolean isApproximation) {
offsetDateTime = OffsetDateTime.ofInstant(calendar.toInstant(), ZoneOffset.UTC);
this.isApproximation = isApproximation;
}
public DateWrapper(@NonNull Calendar date, boolean isApproximation) {
this.date = date;
public DateWrapper(@NonNull OffsetDateTime offsetDateTime) {
this(offsetDateTime, false);
}
public DateWrapper(@NonNull OffsetDateTime offsetDateTime, boolean isApproximation) {
this.offsetDateTime = offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
this.isApproximation = isApproximation;
}
/**
* @return the wrapped date.
* @return the wrapped date/time as a {@link Calendar}.
*
* @deprecated use {@link #offsetDateTime()} instead.
*/
@Deprecated
@NonNull
public Calendar date() {
return date;
return GregorianCalendar.from(offsetDateTime.toZonedDateTime());
}
/**
* @return the wrapped date/time.
*/
@NonNull
public OffsetDateTime offsetDateTime() {
return offsetDateTime;
}
/**
......
......@@ -2,10 +2,11 @@ package org.schabi.newpipe.extractor.localization;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.timeago.PatternsHolder;
import org.schabi.newpipe.extractor.timeago.TimeAgoUnit;
import org.schabi.newpipe.extractor.utils.Parser;
import java.util.Calendar;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit;
import java.util.Collection;
import java.util.Map;
import java.util.regex.Pattern;
......@@ -16,7 +17,7 @@ import java.util.regex.Pattern;
*/
public class TimeAgoParser {
private final PatternsHolder patternsHolder;
private final Calendar consistentNow;
private final OffsetDateTime now;
/**
* Creates a helper to parse upload dates in the format '2 days ago'.
......@@ -28,7 +29,7 @@ public class TimeAgoParser {
*/
public TimeAgoParser(PatternsHolder patternsHolder) {
this.patternsHolder = patternsHolder;
consistentNow = Calendar.getInstance();
now = OffsetDateTime.now(ZoneOffset.UTC);
}
/**
......@@ -42,14 +43,14 @@ public class TimeAgoParser {
* @throws ParsingException if the time unit could not be recognized
*/
public DateWrapper parse(String textualDate) throws ParsingException {
for (Map.Entry<TimeAgoUnit, Map<String, Integer>> caseUnitEntry : patternsHolder.specialCases().entrySet()) {
final TimeAgoUnit timeAgoUnit = caseUnitEntry.getKey();
for (Map.Entry<ChronoUnit, Map<String, Integer>> caseUnitEntry : patternsHolder.specialCases().entrySet()) {
final ChronoUnit chronoUnit = caseUnitEntry.getKey();
for (Map.Entry<String, Integer> caseMapToAmountEntry : caseUnitEntry.getValue().entrySet()) {
final String caseText = caseMapToAmountEntry.getKey();
final Integer caseAmount = caseMapToAmountEntry.getValue();
if (textualDateMatches(textualDate, caseText)) {
return getResultFor(caseAmount, timeAgoUnit);
return getResultFor(caseAmount, chronoUnit);
}
}
}
......@@ -63,8 +64,8 @@ public class TimeAgoParser {
timeAgoAmount = 1;
}
final TimeAgoUnit timeAgoUnit = parseTimeAgoUnit(textualDate);
return getResultFor(timeAgoAmount, timeAgoUnit);
final ChronoUnit chronoUnit = parseChronoUnit(textualDate);
return getResultFor(timeAgoAmount, chronoUnit);
}
private int parseTimeAgoAmount(String textualDate) throws NumberFormatException {
......@@ -72,13 +73,13 @@ public class TimeAgoParser {
return Integer.parseInt(timeValueStr);
}
private TimeAgoUnit parseTimeAgoUnit(String textualDate) throws ParsingException {
for (Map.Entry<TimeAgoUnit, Collection<String>> entry : patternsHolder.asMap().entrySet()) {
final TimeAgoUnit timeAgoUnit = entry.getKey();
private ChronoUnit parseChronoUnit(String textualDate) throws ParsingException {
for (Map.Entry<ChronoUnit, Collection<String>> entry : patternsHolder.asMap().entrySet()) {
final ChronoUnit chronoUnit = entry.getKey();
for (String agoPhrase : entry.getValue()) {
if (textualDateMatches(textualDate, agoPhrase)) {
return timeAgoUnit;
return chronoUnit;
}
}
}
......@@ -112,65 +113,35 @@ public class TimeAgoParser {
}
}
private DateWrapper getResultFor(int timeAgoAmount, TimeAgoUnit timeAgoUnit) {
final Calendar calendarTime = getNow();
private DateWrapper getResultFor(int timeAgoAmount, ChronoUnit chronoUnit) {
OffsetDateTime offsetDateTime = now;
boolean isApproximation = false;
switch (timeAgoUnit) {
switch (chronoUnit) {
case SECONDS:
calendarTime.add(Calendar.SECOND, -timeAgoAmount);
break;
case MINUTES:
calendarTime.add(Calendar.MINUTE, -timeAgoAmount);
break;
case HOURS:
calendarTime.add(Calendar.HOUR_OF_DAY, -timeAgoAmount);
offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit);
break;
case DAYS:
calendarTime.add(Calendar.DAY_OF_MONTH, -timeAgoAmount);
isApproximation = true;
break;
case WEEKS:
calendarTime.add(Calendar.WEEK_OF_YEAR, -timeAgoAmount);
isApproximation = true;
break;
case MONTHS:
calendarTime.add(Calendar.MONTH, -timeAgoAmount);
offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit);
isApproximation = true;
break;
case YEARS:
calendarTime.add(Calendar.YEAR, -timeAgoAmount);
// Prevent `PrettyTime` from showing '12 months ago'.
calendarTime.add(Calendar.DAY_OF_MONTH, -1);
// minusDays is needed to prevent `PrettyTime` from showing '12 months ago'.
offsetDateTime = offsetDateTime.minusYears(timeAgoAmount).minusDays(1);
isApproximation = true;
break;
}
if (isApproximation) {
markApproximatedTime(calendarTime);
offsetDateTime = offsetDateTime.truncatedTo(ChronoUnit.HOURS);
}
return new DateWrapper(calendarTime, isApproximation);
}
private Calendar getNow() {
return (Calendar) consistentNow.clone();
}
/**
* Marks the time as approximated by setting minutes, seconds and milliseconds to 0.
*
* @param calendarTime Time to be marked as approximated
*/
private void markApproximatedTime(Calendar calendarTime) {
calendarTime.set(Calendar.MINUTE, 0);
calendarTime.set(Calendar.SECOND, 0);
calendarTime.set(Calendar.MILLISECOND, 0);
return new DateWrapper(offsetDateTime, isApproximation);
}
}
......@@ -2,28 +2,17 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
import java.time.OffsetDateTime;
import java.time.format.DateTimeParseException;
public final class MediaCCCParsingHelper {
private MediaCCCParsingHelper() { }
public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException {
final Date date;
public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
try {
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e) {
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
}
......@@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.peertube;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
......@@ -12,11 +11,10 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
public class PeertubeParsingHelper {
public static final String START_KEY = "start";
......@@ -34,19 +32,12 @@ public class PeertubeParsingHelper {
}
}
public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException {
final Date date;
public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
try {
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e) {
return OffsetDateTime.ofInstant(Instant.parse(textualUploadDate), ZoneOffset.UTC);
} catch (DateTimeParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
public static Page getNextPage(final String prevPageUrl, final long total) {
......
......@@ -21,16 +21,18 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStr
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
......@@ -95,23 +97,16 @@ public class SoundcloudParsingHelper {
}
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException {
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e1) {
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e1) {
try {
date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(textualUploadDate);
} catch (ParseException e2) {
return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss +0000"));
} catch (DateTimeParseException e2) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + ", " + e1.getMessage(), e2);
}
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
/**
......
......@@ -5,7 +5,6 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.downloader.Response;
......@@ -22,13 +21,18 @@ import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.time.OffsetDateTime;
import java.time.format.DateTimeParseException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.*;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
* Created by Christian Schabesberger on 02.03.16.
......@@ -176,19 +180,12 @@ public class YoutubeParsingHelper {
}
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
final Date date;
public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException {
try {
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e) {
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
public static JsonObject getInitialData(String html) throws ParsingException {
......
......@@ -7,11 +7,9 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
private final Element entryElement;
......@@ -62,19 +60,11 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
final Date date;
try {
final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+00:00");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
date = dateFormat.parse(getTextualUploadDate());
} catch (ParseException e) {
return new DateWrapper(OffsetDateTime.parse(getTextualUploadDate(), DateTimeFormatter.ISO_OFFSET_DATE_TIME));
} catch (DateTimeParseException e) {
throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", e);
}
final Calendar calendar = Calendar.getInstance();
calendar.setTime(date);
return new DateWrapper(calendar);
}
@Override
......
......@@ -43,11 +43,11 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
......@@ -157,23 +157,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try { // Premiered 20 hours ago
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
Calendar parsedTime = timeAgoParser.parse(time).date();
return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime());
OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime();
return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime);
} catch (Exception ignored) {
}
try { // Premiered Feb 21, 2020
Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time);
return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime());
LocalDate localDate = LocalDate.parse(time,
DateTimeFormatter.ofPattern("MMM dd, YYYY", Locale.ENGLISH));
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
} catch (Exception ignored) {
}
}
try {
// TODO: this parses English formatted dates only, we need a better approach to parse the textual date
Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse(
getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")));
return new SimpleDateFormat("yyyy-MM-dd").format(d);
LocalDate localDate = LocalDate.parse(getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")),
DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
} catch (Exception ignored) {
}
throw new ParsingException("Could not get upload date");
......
......@@ -12,11 +12,14 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
......@@ -165,8 +168,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
if (isPremiere()) {
final Date date = getDateFromPremiere().getTime();
return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date);
return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(getDateFromPremiere());
}
final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText"));
......@@ -250,15 +252,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return videoInfo.has("upcomingEventData");
}
private Calendar getDateFromPremiere() throws ParsingException {
private OffsetDateTime getDateFromPremiere() throws ParsingException {
final JsonObject upcomingEventData = videoInfo.getObject("upcomingEventData");
final String startTime = upcomingEventData.getString("startTime");
try {
final long startTimeTimestamp = Long.parseLong(startTime);
final Calendar calendar = Calendar.getInstance();