Commit 725b35ef authored by Christian Schabesberger's avatar Christian Schabesberger
Browse files

merge with changes from master

parents e2b7cb9c f3087553
<?xml version="1.0" encoding="UTF-8"?>
<module external.linked.project.id=":NewPipeExtractor" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$/.." external.system.id="GRADLE" type="JAVA_MODULE" version="4">
<component name="FacetManager">
<facet type="java-gradle" name="Java-Gradle">
<configuration>
<option name="BUILD_FOLDER_PATH" value="$MODULE_DIR$/build" />
<option name="BUILDABLE" value="true" />
</configuration>
</facet>
</component>
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_7" inherit-compiler-output="false">
<output url="file://$MODULE_DIR$/build/classes/java/main" />
<output-test url="file://$MODULE_DIR$/build/classes/java/test" />
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
<excludeFolder url="file://$MODULE_DIR$/.gradle" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" exported="" scope="PROVIDED" name="nanojson-1.1" level="project" />
<orderEntry type="library" exported="" scope="PROVIDED" name="jsoup-1.9.2" level="project" />
<orderEntry type="library" exported="" scope="PROVIDED" name="rhino-1.7.7.1" level="project" />
<orderEntry type="library" exported="" scope="TEST" name="junit-4.12" level="project" />
<orderEntry type="library" exported="" scope="TEST" name="hamcrest-core-1.3" level="project" />
</component>
</module>
\ No newline at end of file
apply plugin: 'java-library'
allprojects {
sourceCompatibility = 1.7
targetCompatibility = 1.7
}
repositories {
jcenter()
}
dependencies {
implementation 'com.github.openjson:openjson:1.0.8'
implementation 'com.grack:nanojson:1.1'
implementation 'org.jsoup:jsoup:1.9.2'
implementation 'org.mozilla:rhino:1.7.7.1'
testImplementation 'junit:junit:4.12'
sourceCompatibility = 1.7
targetCompatibility = 1.7
}
task sourcesJar(type: Jar, dependsOn: classes) {
......
include ':'
rootProject.name = 'NewPipeExtractor'
......@@ -69,9 +69,19 @@ public abstract class ListExtractor extends Extractor {
*/
public final String nextItemsUrl;
public NextItemsResult(List<InfoItem> nextItemsList, String nextItemsUrl) {
/**
* Errors that happened during the extraction
*/
public final List<Throwable> errors;
public NextItemsResult(InfoItemCollector collector, String nextItemsUrl) {
this(collector.getItemList(), nextItemsUrl, collector.getErrors());
}
public NextItemsResult(List<InfoItem> nextItemsList, String nextItemsUrl, List<Throwable> errors) {
this.nextItemsList = nextItemsList;
this.nextItemsUrl = nextItemsUrl;
this.errors = errors;
}
public boolean hasMoreStreams() {
......
......@@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemCollector;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
......@@ -32,6 +34,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
private String suggestion = "";
private StreamInfoItemCollector streamCollector;
private ChannelInfoItemCollector userCollector;
private PlaylistInfoItemCollector playlistCollector;
private SearchResult result = new SearchResult();
......@@ -39,6 +42,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
super(serviceId);
streamCollector = new StreamInfoItemCollector(serviceId);
userCollector = new ChannelInfoItemCollector(serviceId);
playlistCollector = new PlaylistInfoItemCollector(serviceId);
}
public void setSuggestion(String suggestion) {
......@@ -49,6 +53,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
addFromCollector(userCollector);
addFromCollector(streamCollector);
addFromCollector(playlistCollector);
result.suggestion = suggestion;
result.errors = getErrors();
......@@ -74,4 +79,14 @@ public class InfoItemSearchCollector extends InfoItemCollector {
addError(e);
}
}
public void commit(PlaylistInfoItemExtractor extractor) {
try {
result.resultList.add(playlistCollector.extract(extractor));
} catch (FoundAdException ae) {
System.err.println("Found ad");
} catch (Exception e) {
addError(e);
}
}
}
......@@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
import java.util.EnumSet;
/*
* Created by Christian Schabesberger on 10.08.15.
......@@ -27,7 +26,7 @@ import java.util.EnumSet;
public abstract class SearchEngine {
public enum Filter {
STREAM, CHANNEL, PLAYLIST
ANY, STREAM, CHANNEL, PLAYLIST
}
public static class NothingFoundException extends ExtractionException {
......@@ -46,8 +45,6 @@ public abstract class SearchEngine {
return collector;
}
//Result search(String query, int page);
public abstract InfoItemSearchCollector search(
String query, int page, String contentCountry, EnumSet<Filter> filter)
public abstract InfoItemSearchCollector search(String query, int page, String contentCountry, Filter filter)
throws IOException, ExtractionException;
}
......@@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
/*
......@@ -29,8 +28,7 @@ import java.util.List;
*/
public class SearchResult {
public static SearchResult getSearchResult(SearchEngine engine, String query,
int page, String languageCode, EnumSet<SearchEngine.Filter> filter)
public static SearchResult getSearchResult(SearchEngine engine, String query, int page, String languageCode, SearchEngine.Filter filter)
throws IOException, ExtractionException {
SearchResult result = engine
......
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
......@@ -14,7 +16,7 @@ import java.io.IOException;
@SuppressWarnings("WeakerAccess")
public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId;
private JSONObject user;
private JsonObject user;
public SoundcloudChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
......@@ -29,16 +31,16 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
"?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
user = new JSONObject(response);
try {
user = JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}
@Override
public String getCleanUrl() {
try {
return user.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
return user.isString("permalink_url") ? user.getString("permalink_url") : getOriginalUrl();
}
@Override
......@@ -53,16 +55,12 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public String getAvatarUrl() {
return user.optString("avatar_url");
return user.getString("avatar_url");
}
@Override
public String getBannerUrl() throws ParsingException {
try {
return user.getJSONObject("visuals").getJSONArray("visuals").getJSONObject(0).getString("visual_url");
} catch (Exception e) {
throw new ParsingException("Could not get Banner", e);
}
return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url", "");
}
@Override
......@@ -72,12 +70,12 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public long getSubscriberCount() {
return user.optLong("followers_count");
return user.getNumber("followers_count", 0).longValue();
}
@Override
public String getDescription() throws ParsingException {
return user.optString("description");
return user.getString("description", "");
}
@Override
......@@ -102,6 +100,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
return new NextItemsResult(collector, nextStreamsUrl);
}
}
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private JSONObject searchResult;
private JsonObject searchResult;
public SoundcloudChannelInfoItemExtractor(JSONObject searchResult) {
public SoundcloudChannelInfoItemExtractor(JsonObject searchResult) {
this.searchResult = searchResult;
}
......@@ -22,21 +22,21 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
@Override
public String getThumbnailUrl() {
return searchResult.optString("avatar_url");
return searchResult.getString("avatar_url", "");
}
@Override
public long getSubscriberCount() {
return searchResult.optLong("followers_count");
return searchResult.getNumber("followers_count", 0).longValue();
}
@Override
public long getStreamCount() {
return searchResult.optLong("track_count");
return searchResult.getNumber("track_count", 0).longValue();
}
@Override
public String getDescription() {
return searchResult.optString("description");
return searchResult.getString("description", "");
}
}
......@@ -40,7 +40,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl, true);
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
return new NextItemsResult(collector, nextStreamsUrl);
}
@Override
......
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONArray;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
......@@ -65,12 +67,16 @@ public class SoundcloudParsingHelper {
* Call the endpoint "/resolve" of the api.<br/>
* See https://developers.soundcloud.com/docs/api/reference#resolve
*/
public static JSONObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException {
public static JsonObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException {
String apiUrl = "https://api.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, "UTF-8")
+ "&client_id=" + clientId();
return new JSONObject(NewPipe.getDownloader().download(apiUrl));
try {
return JsonParser.object().from(NewPipe.getDownloader().download(apiUrl));
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}
/**
......@@ -123,15 +129,16 @@ public class SoundcloudParsingHelper {
*/
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl);
JSONObject responseObject = new JSONObject(response);
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
if (charts) {
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i).getJSONObject("track")));
} else {
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i)));
}
JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
JsonArray responseCollection = responseObject.getArray("collection");
for (Object o : responseCollection) {
if (o instanceof JsonObject) collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) o));
}
String nextStreamsUrl;
......
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
......@@ -13,7 +16,7 @@ import java.io.IOException;
@SuppressWarnings("WeakerAccess")
public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private String playlistId;
private JSONObject playlist;
private JsonObject playlist;
public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
......@@ -29,16 +32,16 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
"&representation=compact";
String response = dl.download(apiUrl);
playlist = new JSONObject(response);
try {
playlist = JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}
@Override
public String getCleanUrl() {
try {
return playlist.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
return playlist.isString("permalink_url") ? playlist.getString("permalink_url") : getOriginalUrl();
}
@Override
......@@ -48,12 +51,12 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Override
public String getName() {
return playlist.optString("title");
return playlist.getString("title");
}
@Override
public String getThumbnailUrl() {
return playlist.optString("artwork_url");
return playlist.getString("artwork_url");
}
@Override
......@@ -63,22 +66,22 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderUrl() {
return playlist.getJSONObject("user").getString("permalink_url");
return playlist.getObject("user").getString("permalink_url", "");
}
@Override
public String getUploaderName() {
return playlist.getJSONObject("user").getString("username");
return playlist.getObject("user").getString("username", "");
}
@Override
public String getUploaderAvatarUrl() {
return playlist.getJSONObject("user").getString("avatar_url");
return playlist.getObject("user", new JsonObject()).getString("avatar_url", "");
}
@Override
public long getStreamCount() {
return playlist.getLong("track_count");
return playlist.getNumber("track_count", 0).longValue();
}
@Override
......@@ -104,6 +107,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
return new NextItemsResult(collector, nextStreamsUrl);
}
}
package org.schabi.newpipe.extractor.services.soundcloud;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
private static final String USER_KEY = "user";
private static final String AVATAR_URL_KEY = "avatar_url";
private static final String ARTWORK_URL_KEY = "artwork_url";
private JsonObject searchResult;
public SoundcloudPlaylistInfoItemExtractor(JsonObject searchResult) {
this.searchResult = searchResult;
}
@Override
public String getName() throws ParsingException {
return searchResult.getString("title");
}
@Override
public String getUrl() throws ParsingException {
return searchResult.getString("permalink_url");
}
@Override
public String getThumbnailUrl() throws ParsingException {
// Over-engineering at its finest
if (searchResult.isString(ARTWORK_URL_KEY)) {
final String artworkUrl = searchResult.getString(ARTWORK_URL_KEY, "");
if (!artworkUrl.isEmpty()) return artworkUrl;
}
try {
// Look for artwork url inside the track list
for (Object track : searchResult.getArray("tracks")) {
final JsonObject trackObject = (JsonObject) track;
// First look for track artwork url
if (trackObject.isString(ARTWORK_URL_KEY)) {
final String url = trackObject.getString(ARTWORK_URL_KEY, "");
if (!url.isEmpty()) return url;
}
// Then look for track creator avatar url
final JsonObject creator = trackObject.getObject(USER_KEY, new JsonObject());
final String creatorAvatar = creator.getString(AVATAR_URL_KEY, "");
if (!creatorAvatar.isEmpty()) return creatorAvatar;
}
} catch (Exception ignored) {
// Try other method
}
try {
// Last resort, use user avatar url. If still not found, then throw exception.
return searchResult.getObject(USER_KEY).getString(AVATAR_URL_KEY, "");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist thumbnail url", e);
}
}
@Override
public String getUploaderName() throws ParsingException {
try {
return searchResult.getObject(USER_KEY).getString("username");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist uploader", e);
}
}
@Override
public long getStreamCount() throws ParsingException {
return searchResult.getNumber("track_count", 0).longValue();
}
}
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONArray;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.EnumSet;
public class SoundcloudSearchEngine extends SearchEngine {
public static final String CHARSET_UTF_8 = "UTF-8";
......@@ -20,17 +22,27 @@ public class SoundcloudSearchEngine extends SearchEngine {
}
@Override
public InfoItemSearchCollector search(String query, int page, String languageCode, EnumSet<Filter> filter) throws IOException, ExtractionException {
public InfoItemSearchCollector search(String query, int page, String languageCode, Filter filter) throws IOException, ExtractionException {
InfoItemSearchCollector collector = getInfoItemSearchCollector();
Downloader downloader = NewPipe.getDownloader();
Downloader dl = NewPipe.getDownloader();
String url = "https://api-v2.soundcloud.com/search";