Commit 9184fc50 authored by Mauricio Colli's avatar Mauricio Colli
Browse files

Store errors that can happen during extraction of the next page

- Closes #24
parent 82824cdd
......@@ -69,9 +69,19 @@ public abstract class ListExtractor extends Extractor {
*/
public final String nextItemsUrl;
public NextItemsResult(List<InfoItem> nextItemsList, String nextItemsUrl) {
/**
* Errors that happened during the extraction
*/
public final List<Throwable> errors;
public NextItemsResult(InfoItemCollector collector, String nextItemsUrl) {
this(collector.getItemList(), nextItemsUrl, collector.getErrors());
}
public NextItemsResult(List<InfoItem> nextItemsList, String nextItemsUrl, List<Throwable> errors) {
this.nextItemsList = nextItemsList;
this.nextItemsUrl = nextItemsUrl;
this.errors = errors;
}
public boolean hasMoreStreams() {
......
......@@ -102,6 +102,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
return new NextItemsResult(collector, nextStreamsUrl);
}
}
......@@ -104,6 +104,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
return new NextItemsResult(collector, nextStreamsUrl);
}
}
......@@ -153,7 +153,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
return new NextItemsResult(collector, nextStreamsUrl);
}
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
......
......@@ -157,7 +157,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
return new NextItemsResult(collector, nextStreamsUrl);
}
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
......
......@@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
......@@ -70,7 +71,9 @@ public class SoundcloudChannelExtractorTest {
public void testGetNextStreams() throws Exception {
// Setup the streams
extractor.getStreams();
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams();
assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty());
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
}
......
......@@ -4,6 +4,7 @@ import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import static org.junit.Assert.*;
......@@ -79,4 +80,15 @@ public class SoundcloudPlaylistExtractorTest {
extractor.getStreams();
assertTrue("extractor didn't have more streams", !extractor.hasMoreStreams());
}
@Test(expected = ExtractionException.class)
public void testGetNextStreamsNonExistent() throws Exception {
// Setup the streams
extractor.getStreams();
// This playlist don't have more streams, it should throw an error
extractor.getNextStreams();
fail("Expected exception wasn't thrown");
}
}
......@@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
......@@ -99,7 +100,9 @@ public class YoutubeChannelExtractorTest {
public void testGetNextStreams() throws Exception {
// Setup the streams
extractor.getStreams();
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams();
assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty());
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
}
......
......@@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
......@@ -89,7 +90,9 @@ public class YoutubePlaylistExtractorTest {
public void testGetNextStreams() throws Exception {
// Setup the streams
extractor.getStreams();
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams();
assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty());
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment