Commit 4086715a authored by TobiGr's avatar TobiGr
Browse files

Merge branch 'dev'

parents 04af5757 8a79b140
- [ ] I carefully read the [contribution guidelines](https://github.com/TeamNewPipe/NewPipe/blob/HEAD/.github/CONTRIBUTING.md) and agree to them.
- [ ] I did test the API against [NewPipe](https://github.com/TeamNewPipe/NewPipe).
- [ ] I agree to ASAP create a PULL request for [NewPipe](https://github.com/TeamNewPipe/NewPipe) for making in compatible when I changed the api.
- [ ] I have tested the API against [NewPipe](https://github.com/TeamNewPipe/NewPipe).
- [ ] I agree to create a pull request for [NewPipe](https://github.com/TeamNewPipe/NewPipe) as soon as possible to make it compatible with the changed API.
# NewPipe Extractor
[![Build Status](https://travis-ci.org/TeamNewPipe/NewPipeExtractor.svg?branch=master)](https://travis-ci.org/TeamNewPipe/NewPipeExtractor) [![JIT Pack Badge](https://jitpack.io/v/TeamNewPipe/NewPipeExtractor.svg)](https://jitpack.io/#TeamNewPipe/NewPipeExtractor) [Documentation](https://teamnewpipe.github.io/documentation/)
[![Build Status](https://travis-ci.org/TeamNewPipe/NewPipeExtractor.svg?branch=master)](https://travis-ci.org/TeamNewPipe/NewPipeExtractor) [![JIT Pack Badge](https://jitpack.io/v/TeamNewPipe/NewPipeExtractor.svg)](https://jitpack.io/#TeamNewPipe/NewPipeExtractor) [JDoc](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/)[Documentation](https://teamnewpipe.github.io/documentation/)
NewPipe Extractor is a library for extracting things from streaming sites. It is a core component of [NewPipe](https://github.com/TeamNewPipe/NewPipe), but could be used independently.
......@@ -11,11 +11,21 @@ NewPipe Extractor is available at JitPack's Maven repo.
If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps:
1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
2. Add `compile 'com.github.TeamNewPipe:NewPipeExtractor:v0.11.0'`the `dependencies` in your `build.gradle`. Replace `v0.11.0` with the latest release.
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.19.0'`the `dependencies` in your `build.gradle`. Replace `v0.19.0` with the latest release.
### Testing changes
To test changes quickly you can build the library locally. Using the local Maven repository is a good approach, here's a gist of how to use it:
To test changes quickly you can build the library locally. A good approach would be to add something like the following to your `settings.gradle`:
```groovy
includeBuild('../NewPipeExtractor') {
dependencySubstitution {
substitute module('com.github.TeamNewPipe:NewPipeExtractor') with project(':extractor')
}
}
```
Another approach would be to use the local Maven repository, here's a gist of how to use it:
1. Add `mavenLocal()` in your project `repositories` list (usually as the first entry to give priority above the others).
2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`).
......
......@@ -5,11 +5,12 @@ allprojects {
sourceCompatibility = 1.7
targetCompatibility = 1.7
version 'v0.19.0'
version 'v0.19.4'
group 'com.github.TeamNewPipe'
repositories {
jcenter()
maven { url "https://jitpack.io" }
}
}
......
dependencies {
implementation project(':timeago-parser')
implementation 'com.grack:nanojson:1.1'
implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
implementation 'org.jsoup:jsoup:1.9.2'
implementation 'org.mozilla:rhino:1.7.7.1'
implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
implementation 'org.nibor.autolink:autolink:0.8.0'
testImplementation 'junit:junit:4.12'
}
\ No newline at end of file
}
......@@ -3,16 +3,33 @@ package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
/**
* Base class to extractors that have a list (e.g. playlists, users).
*/
public abstract class ListExtractor<R extends InfoItem> extends Extractor {
/**
* Constant that should be returned whenever
* a list has an unknown number of items.
*/
public static final long ITEM_COUNT_UNKNOWN = -1;
/**
* Constant that should be returned whenever a list has an
* infinite number of items. For example a YouTube mix.
*/
public static final long ITEM_COUNT_INFINITE = -2;
/**
* Constant that should be returned whenever a list
* has an unknown number of items bigger than 100.
*/
public static final long ITEM_COUNT_MORE_THAN_100 = -3;
public ListExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
......
package org.schabi.newpipe.extractor.exceptions;
public class ContentNotSupportedException extends ParsingException {
public ContentNotSupportedException(String message) {
super(message);
}
public ContentNotSupportedException(String message, Throwable cause) {
super(message, cause);
}
}
......@@ -6,7 +6,12 @@ import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor;
......@@ -21,19 +26,17 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
public class MediaCCCService extends StreamingService {
public MediaCCCService(int id) {
public MediaCCCService(final int id) {
super(id, "MediaCCC", asList(AUDIO, VIDEO));
}
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler query) {
public SearchExtractor getSearchExtractor(final SearchQueryHandler query) {
return new MediaCCCSearchExtractor(this, query);
}
......@@ -58,17 +61,17 @@ public class MediaCCCService extends StreamingService {
}
@Override
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) {
public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) {
return new MediaCCCStreamExtractor(this, linkHandler);
}
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) {
return new MediaCCCConferenceExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return null;
}
......@@ -85,9 +88,9 @@ public class MediaCCCService extends StreamingService {
try {
list.addKioskEntry(new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String kioskId) throws ExtractionException, IOException {
public KioskExtractor createNewKiosk(final StreamingService streamingService,
final String url, final String kioskId)
throws ExtractionException {
return new MediaCCCConferenceKiosk(MediaCCCService.this,
new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), kioskId);
}
......@@ -111,8 +114,7 @@ public class MediaCCCService extends StreamingService {
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
public CommentsExtractor getCommentsExtractor(final ListLinkHandler linkHandler) {
return null;
}
......
......@@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
......@@ -14,45 +15,46 @@ import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.Medi
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
public class MediaCCCConferenceExtractor extends ChannelExtractor {
import javax.annotation.Nonnull;
public class MediaCCCConferenceExtractor extends ChannelExtractor {
private JsonObject conferenceData;
public MediaCCCConferenceExtractor(StreamingService service, ListLinkHandler linkHandler) {
public MediaCCCConferenceExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public String getAvatarUrl() throws ParsingException {
public String getAvatarUrl() {
return conferenceData.getString("logo_url");
}
@Override
public String getBannerUrl() throws ParsingException {
public String getBannerUrl() {
return conferenceData.getString("logo_url");
}
@Override
public String getFeedUrl() throws ParsingException {
public String getFeedUrl() {
return null;
}
@Override
public long getSubscriberCount() throws ParsingException {
public long getSubscriberCount() {
return -1;
}
@Override
public String getDescription() throws ParsingException {
public String getDescription() {
return null;
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getInitialPage() {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray events = conferenceData.getArray("events");
for (int i = 0; i < events.size(); i++) {
......@@ -62,17 +64,18 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
public String getNextPageUrl() {
return null;
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl) {
return null;
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
try {
conferenceData = JsonParser.object().from(downloader.get(getUrl()).responseBody());
} catch (JsonParserException jpe) {
......@@ -88,7 +91,7 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
@Nonnull
@Override
public String getOriginalUrl() throws ParsingException {
public String getOriginalUrl() {
return "https://media.ccc.de/c/" + conferenceData.getString("acronym");
}
}
......@@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
......@@ -14,22 +15,22 @@ import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCConferenceInfoItemExtractor;
import javax.annotation.Nonnull;
import java.io.IOException;
public class MediaCCCConferenceKiosk extends KioskExtractor<ChannelInfoItem> {
import javax.annotation.Nonnull;
public class MediaCCCConferenceKiosk extends KioskExtractor<ChannelInfoItem> {
private JsonObject doc;
public MediaCCCConferenceKiosk(StreamingService streamingService,
ListLinkHandler linkHandler,
String kioskId) {
public MediaCCCConferenceKiosk(final StreamingService streamingService,
final ListLinkHandler linkHandler,
final String kioskId) {
super(streamingService, linkHandler, kioskId);
}
@Nonnull
@Override
public InfoItemsPage<ChannelInfoItem> getInitialPage() throws IOException, ExtractionException {
public InfoItemsPage<ChannelInfoItem> getInitialPage() {
JsonArray conferences = doc.getArray("conferences");
ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(getServiceId());
for (int i = 0; i < conferences.size(); i++) {
......@@ -40,18 +41,20 @@ public class MediaCCCConferenceKiosk extends KioskExtractor<ChannelInfoItem> {
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
public String getNextPageUrl() {
return "";
}
@Override
public InfoItemsPage<ChannelInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
public InfoItemsPage<ChannelInfoItem> getPage(final String pageUrl) {
return InfoItemsPage.emptyPage();
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String site = downloader.get(getLinkHandler().getUrl(), getExtractorLocalization()).responseBody();
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String site = downloader.get(getLinkHandler().getUrl(), getExtractorLocalization())
.responseBody();
try {
doc = JsonParser.object().from(site);
} catch (JsonParserException jpe) {
......
......@@ -7,11 +7,10 @@ import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
public class MediaCCCParsingHelper {
private MediaCCCParsingHelper() {
}
public final class MediaCCCParsingHelper {
private MediaCCCParsingHelper() { }
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException {
Date date;
try {
date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate);
......
......@@ -4,31 +4,34 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.List;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.*;
import javax.annotation.Nonnull;
public class MediaCCCSearchExtractor extends SearchExtractor {
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.ALL;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.CONFERENCES;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.EVENTS;
public class MediaCCCSearchExtractor extends SearchExtractor {
private JsonObject doc;
private MediaCCCConferenceKiosk conferenceKiosk;
public MediaCCCSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
public MediaCCCSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
super(service, linkHandler);
try {
conferenceKiosk = new MediaCCCConferenceKiosk(service,
......@@ -40,13 +43,13 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
}
@Override
public String getSearchSuggestion() throws ParsingException {
public String getSearchSuggestion() {
return null;
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
public InfoItemsPage<InfoItem> getInitialPage() {
final InfoItemsSearchCollector searchItems = new InfoItemsSearchCollector(getServiceId());
if (getLinkHandler().getContentFilters().contains(CONFERENCES)
......@@ -70,17 +73,18 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
public String getNextPageUrl() {
return "";
}
@Override
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
public InfoItemsPage<InfoItem> getPage(final String pageUrl) {
return InfoItemsPage.emptyPage();
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
if (getLinkHandler().getContentFilters().contains(EVENTS)
|| getLinkHandler().getContentFilters().contains(ALL)
|| getLinkHandler().getContentFilters().isEmpty()) {
......@@ -95,44 +99,45 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
}
if (getLinkHandler().getContentFilters().contains(CONFERENCES)
|| getLinkHandler().getContentFilters().contains(ALL)
|| getLinkHandler().getContentFilters().isEmpty())
|| getLinkHandler().getContentFilters().isEmpty()) {
conferenceKiosk.fetchPage();
}
}
private void searchConferences(String searchString,
List<ChannelInfoItem> channelItems,
InfoItemsSearchCollector collector) {
private void searchConferences(final String searchString,
final List<ChannelInfoItem> channelItems,
final InfoItemsSearchCollector collector) {
for (final ChannelInfoItem item : channelItems) {
if (item.getName().toUpperCase().contains(
searchString.toUpperCase())) {
collector.commit(new ChannelInfoItemExtractor() {
@Override
public String getDescription() throws ParsingException {
public String getDescription() {
return item.getDescription();
}
@Override
public long getSubscriberCount() throws ParsingException {
public long getSubscriberCount() {
return item.getSubscriberCount();
}
@Override
public long getStreamCount() throws ParsingException {
public long getStreamCount() {
return item.getStreamCount();
}
@Override
public String getName() throws ParsingException {
public String getName() {
return item.getName();
}
@Override
public String getUrl() throws ParsingException {
public String getUrl() {
return item.getUrl();
}
@Override
public String getThumbnailUrl() throws ParsingException {
public String getThumbnailUrl() {
return item.getThumbnailUrl();
}
});
......
......@@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
......@@ -11,27 +12,34 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
public class MediaCCCStreamExtractor extends StreamExtractor {
import javax.annotation.Nonnull;
public class MediaCCCStreamExtractor extends StreamExtractor {
private JsonObject data;
private JsonObject conferenceData;
public MediaCCCStreamExtractor(StreamingService service, LinkHandler linkHandler) {
public MediaCCCStreamExtractor(final StreamingService service, final LinkHandler linkHandler) {
super(service, linkHandler);
}
@Nonnull
@Override
public String getTextualUploadDate() throws ParsingException {
public String getTextualUploadDate() {
return data.getString("release_date");
}
......@@ -43,79 +51,79 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getThumbnailUrl() throws ParsingException {
public String getThumbnailUrl() {
return data.getString("thumb_url");
}
@Nonnull
@Override
public Description getDescription() throws ParsingException {
public Description getDescription() {
return new Description(data.