Commit 0c632e19 authored by Christian Schabesberger's avatar Christian Schabesberger Committed by GitHub
Browse files

Merge pull request #27 from TeamNewPipe/kiosk

add Kiosk support
parents 7ae274b2 f64c82f0
package org.schabi.newpipe.extractor;
/*
* Created by the-scrabi on 11.02.17.
* Created by Christian Schabesberger on 11.02.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* InfoItem.java is part of NewPipe.
......
......@@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
......@@ -48,6 +49,7 @@ public abstract class StreamingService {
public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException;
public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
public abstract KioskList getKioskList() throws ExtractionException;
public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException {
return getChannelExtractor(url, null);
......
package org.schabi.newpipe.extractor.kiosk;
/*
* Created by Christian Schabesberger on 12.08.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* KioskExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.IOException;
public abstract class KioskExtractor extends ListExtractor {
private String contentCountry = null;
public KioskExtractor(StreamingService streamingService,
String url,
String nextStreamsUrl)
throws IOException, ExtractionException {
super(streamingService, url, nextStreamsUrl);
this.contentCountry = contentCountry;
}
/**
* For certain Websites the content of a kiosk will be different depending
* on the country you want to poen the website in. Therefore you should
* set the contentCountry.
* @param contentCountry Set the country decoded as Country Code: http://www.1728.org/countries.htm
*/
public void setContentCountry(String contentCountry) {
this.contentCountry = contentCountry;
}
/**
* Returns the type of the kiosk.
* eg. Trending, Top & Hot, Top last 24 hours
* @return type of kiosk
*/
public abstract String getType() throws ParsingException;
@Override
public String getId() throws ParsingException {
return getType();
}
@Override
public String getName() throws ParsingException {
return getType();
}
public String getContentCountry() {
return contentCountry;
}
}
package org.schabi.newpipe.extractor.kiosk;
/*
* Created by Christian Schabesberger on 12.08.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* KioskInfo.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException;
public class KioskInfo extends ListInfo {
public String type;
public static KioskInfo getInfo(String url,
String contentCountry) throws IOException, ExtractionException {
return getInfo(NewPipe.getServiceByUrl(url), url, contentCountry);
}
public static KioskInfo getInfo(ServiceList serviceItem,
String url,
String contentContry) throws IOException, ExtractionException {
return getInfo(serviceItem.getService(), url, contentContry);
}
public static KioskInfo getInfo(StreamingService service,
String url,
String contentCountry) throws IOException, ExtractionException {
KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtryctorByUrl(url);
return getInfo(extractor, contentCountry);
}
public static KioskInfo getInfo(KioskExtractor extractor,
String contentCountry) throws IOException, ExtractionException {
KioskInfo info = new KioskInfo();
extractor.setContentCountry(contentCountry);
extractor.fetchPage();
info.type = extractor.getType();
info.name = extractor.getName();
info.id = extractor.getId();
try {
StreamInfoItemCollector c = extractor.getStreams();
info.related_streams = c.getItemList();
info.errors.addAll(c.getErrors());
} catch (Exception e) {
info.errors.add(e);
}
return info;
}
}
package org.schabi.newpipe.extractor.kiosk;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
public class KioskList {
private int service_id;
private HashMap<String, KioskEntry> kioskList = new HashMap<>();
private class KioskEntry {
public KioskEntry(KioskExtractor e, UrlIdHandler h) {
extractor = e;
handler = h;
}
KioskExtractor extractor;
UrlIdHandler handler;
}
public KioskList(int service_id) {
this.service_id = service_id;
}
public void addKioskEntry(KioskExtractor extractor, UrlIdHandler handler)
throws Exception {
if(kioskList.get(extractor.getType()) != null) {
throw new Exception("Kiosk with type " + extractor.getType() + " already exists.");
}
kioskList.put(extractor.getType(), new KioskEntry(extractor, handler));
}
public KioskExtractor getExtractorByType(String kioskType) throws ExtractionException {
KioskEntry ke = kioskList.get(kioskType);
if(ke == null) {
throw new ExtractionException("No kiosk found with the type: " + kioskType);
} else {
return ke.extractor;
}
}
public Set<String> getAvailableKisokTypes() {
return kioskList.keySet();
}
public KioskExtractor getExtryctorByUrl(String url) throws ExtractionException {
for(Map.Entry<String, KioskEntry> e : kioskList.entrySet()) {
KioskEntry ke = e.getValue();
if(ke.handler.acceptUrl(url)) {
return getExtractorByType(e.getKey());
}
}
throw new ExtractionException("Could not find a kiosk that fits to the url: " + url);
}
}
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
public class SoundcloudChartsExtractor extends KioskExtractor {
private String url;
public SoundcloudChartsExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
this.url = url;
}
@Override
public void fetchPage() {
}
@Override
public String getType() throws ParsingException {
return getUrlIdHandler().getId(url);
}
@Override
public UrlIdHandler getUrlIdHandler() {
return new SoundcloudChartsUrlIdHandler();
}
@Override
public NextItemsResult getNextStreams() throws IOException, ExtractionException {
if (!hasMoreStreams()) {
throw new ExtractionException("Chart doesn't have more streams");
}
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl, true);
return new NextItemsResult(collector, nextStreamsUrl);
}
@Override
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/charts" +
"?genre=soundcloud:genres:all-music" +
"&client_id=" + SoundcloudParsingHelper.clientId();
if (getType().equals("Top 50")) {
apiUrl += "&kind=top";
} else {
apiUrl += "&kind=new";
}
List<String> supportedCountries = Arrays.asList("AU", "CA", "FR", "DE", "IE", "NL", "NZ", "GB", "US");
String contentCountry = getContentCountry();
if (supportedCountries.contains(contentCountry)) {
apiUrl += "&region=soundcloud:regions:" + contentCountry;
}
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
return collector;
}
}
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.utils.Parser;
public class SoundcloudChartsUrlIdHandler implements UrlIdHandler {
public String getUrl(String id) {
if (id.equals("Top 50")) {
return "https://soundcloud.com/charts/top";
} else {
return "https://soundcloud.com/charts/new";
}
}
@Override
public String getId(String url) {
if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) {
return "Top 50";
} else {
return "New & hot";
}
}
@Override
public String cleanUrl(String url) {
if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) {
return "https://soundcloud.com/charts/top";
} else {
return "https://soundcloud.com/charts/new";
}
}
@Override
public boolean acceptUrl(String url) {
return Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$", url.toLowerCase());
}
}
......@@ -127,7 +127,7 @@ public class SoundcloudParsingHelper {
*
* @return the next streams url, empty if don't have
*/
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl);
JsonObject responseObject;
try {
......@@ -151,4 +151,8 @@ public class SoundcloudParsingHelper {
return nextStreamsUrl;
}
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException {
return getStreamsFromApi(collector, apiUrl, false);
}
}
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import java.io.IOException;
public class SoundcloudService extends StreamingService {
public SoundcloudService(int id, String name) {
......@@ -57,4 +58,20 @@ public class SoundcloudService extends StreamingService {
public SuggestionExtractor getSuggestionExtractor() {
return new SoundcloudSuggestionExtractor(getServiceId());
}
@Override
public KioskList getKioskList() throws ExtractionException {
KioskList list = new KioskList(getServiceId());
// add kiosks here e.g.:
SoundcloudChartsUrlIdHandler h = new SoundcloudChartsUrlIdHandler();
try {
list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("Top 50"), null), h);
list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("New & hot"), null), h);
} catch (Exception e) {
throw new ExtractionException(e);
}
return list;
}
}
......@@ -30,12 +30,13 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
@Override
public String getUploaderName() {
return searchResult.getObject("user").getString("username");
//return searchResult.getObject("user").getString("username");
return searchResult.getObject("track").getObject("user").getString("username");
}
@Override
public String getUploadDate() throws ParsingException {
return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at"));
return SoundcloudParsingHelper.toDateString(searchResult.getObject("track").getString("created_at"));
}
@Override
......
......@@ -5,6 +5,7 @@ import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
......@@ -78,4 +79,19 @@ public class YoutubeService extends StreamingService {
public SuggestionExtractor getSuggestionExtractor() {
return new YoutubeSuggestionExtractor(getServiceId());
}
@Override
public KioskList getKioskList() throws ExtractionException {
KioskList list = new KioskList(getServiceId());
// add kiosks here e.g.:
YoutubeTrendingUrlIdHandler h = new YoutubeTrendingUrlIdHandler();
try {
list.addKioskEntry(new YoutubeTrendingExtractor(this, h.getUrl(""), null), h);
} catch (Exception e) {
throw new ExtractionException(e);
}
return list;
}
}
package org.schabi.newpipe.extractor.services.youtube;
/*
* Created by Christian Schabesberger on 12.08.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* YoutubeTrendingExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException;
public class YoutubeTrendingExtractor extends KioskExtractor {
private Document doc;
public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl)
throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
}
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader downloader = NewPipe.getDownloader();
final String contentCountry = getContentCountry();
String url = getCleanUrl();
if(contentCountry != null && !contentCountry.isEmpty()) {
url += "?gl=" + contentCountry;
}
String pageContent = downloader.download(url);
doc = Jsoup.parse(pageContent, url);
}
@Override
public String getType() {
return "Trending";
}
@Override
public UrlIdHandler getUrlIdHandler() {
return new YoutubeTrendingUrlIdHandler();
}
@Override
public ListExtractor.NextItemsResult getNextStreams() {
return null;
}
@Override
public StreamInfoItemCollector getStreams() throws ParsingException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Element ul = doc.select("ul[class*=\"expanded-shelf-content-list\"]").first();
for(final Element li : ul.children()) {
final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first();
collector.commit(new YoutubeStreamInfoItemExtractor(li) {
@Override
public String getUrl() throws ParsingException {
try {
Element dl = el.select("h3").first().select("a").first();
return dl.attr("abs:href");
} catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e);
}
}
@Override
public String getName() throws ParsingException {
try {
Element dl = el.select("h3").first().select("a").first();
return dl.text();
} catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e);
}
}
@Override
public String getUploaderName() throws ParsingException {
try {
Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first();
return uploaderEl.select("a").text();
} catch (Exception e) {
throw new ParsingException("Could not get Uploader name");
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
try {
String url;
Element te = li.select("span[class=\"yt-thumb-simple\"]").first()
.select("img").first();
url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (url.contains(".gif")) {
url = te.attr("abs:data-thumb");
}
return url;
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);