Unverified Commit e4874d55 authored by Tobias Groza's avatar Tobias Groza Committed by GitHub
Browse files

Merge pull request #296 from mauriciocolli/fix-search-errors-detection

Fix search errors detection and refactor search tests
parents 8a9ae32e 0a20c53f
......@@ -11,6 +11,10 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/*
* Created by Christian Schabesberger on 12.02.17.
*
......@@ -48,13 +52,31 @@ public class InfoItemsSearchCollector extends InfoItemsCollector<InfoItem, InfoI
private final ChannelInfoItemsCollector userCollector;
private final PlaylistInfoItemsCollector playlistCollector;
InfoItemsSearchCollector(int serviceId) {
public InfoItemsSearchCollector(int serviceId) {
super(serviceId);
streamCollector = new StreamInfoItemsCollector(serviceId);
userCollector = new ChannelInfoItemsCollector(serviceId);
playlistCollector = new PlaylistInfoItemsCollector(serviceId);
}
@Override
public List<Throwable> getErrors() {
final List<Throwable> errors = new ArrayList<>(super.getErrors());
errors.addAll(streamCollector.getErrors());
errors.addAll(userCollector.getErrors());
errors.addAll(playlistCollector.getErrors());
return Collections.unmodifiableList(errors);
}
@Override
public void reset() {
super.reset();
streamCollector.reset();
userCollector.reset();
playlistCollector.reset();
}
@Override
public InfoItem extract(InfoItemExtractor extractor) throws ParsingException {
// Use the corresponding collector for each item extractor type
......
......@@ -17,11 +17,8 @@ public abstract class SearchExtractor extends ListExtractor<InfoItem> {
}
}
private final InfoItemsSearchCollector collector;
public SearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
super(service, linkHandler);
collector = new InfoItemsSearchCollector(service.getServiceId());
}
public String getSearchString() {
......@@ -30,10 +27,6 @@ public abstract class SearchExtractor extends ListExtractor<InfoItem> {
public abstract String getSearchSuggestion() throws ParsingException;
protected InfoItemsSearchCollector getInfoItemSearchCollector() {
return collector;
}
@Override
public SearchQueryHandler getLinkHandler() {
return (SearchQueryHandler) super.getLinkHandler();
......
......@@ -47,8 +47,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
InfoItemsSearchCollector searchItems = getInfoItemSearchCollector();
searchItems.reset();
final InfoItemsSearchCollector searchItems = new InfoItemsSearchCollector(getServiceId());
if (getLinkHandler().getContentFilters().contains(CONFERENCES)
|| getLinkHandler().getContentFilters().contains(ALL)
......
......@@ -47,8 +47,7 @@ public class PeertubeSearchExtractor extends SearchExtractor {
}
private InfoItemsCollector<InfoItem, InfoItemExtractor> collectStreamsFrom(JsonObject json) throws ParsingException {
final InfoItemsSearchCollector collector = getInfoItemSearchCollector();
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
JsonArray contents;
try {
......
......@@ -78,8 +78,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
}
private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(JsonArray searchCollection) {
final InfoItemsSearchCollector collector = getInfoItemSearchCollector();
collector.reset();
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
for (Object result : searchCollection) {
if (!(result instanceof JsonObject)) continue;
......
......@@ -70,8 +70,14 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public long getSubscriberCount() throws ParsingException {
try {
String subscribers = getTextFromObject(channelInfoItem.getObject("subscriberCountText"));
return Utils.mixedNumberWordToLong(subscribers);
final JsonObject subscriberCountObject = channelInfoItem.getObject("subscriberCountText");
if (subscriberCountObject == null) {
// Subscription count is not available for this channel item.
return -1;
}
return Utils.mixedNumberWordToLong(getTextFromObject(subscriberCountObject));
} catch (Exception e) {
throw new ParsingException("Could not get subscriber count", e);
}
......@@ -80,7 +86,14 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public long getStreamCount() throws ParsingException {
try {
return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(channelInfoItem.getObject("videoCountText"))));
final JsonObject videoCountObject = channelInfoItem.getObject("videoCountText");
if (videoCountObject == null) {
// Video count is not available, channel probably has no public uploads.
return -1;
}
return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(videoCountObject)));
} catch (Exception e) {
throw new ParsingException("Could not get stream count", e);
}
......@@ -89,7 +102,14 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public String getDescription() throws ParsingException {
try {
return getTextFromObject(channelInfoItem.getObject("descriptionSnippet"));
final JsonObject descriptionObject = channelInfoItem.getObject("descriptionSnippet");
if (descriptionObject == null) {
// Channel have no description.
return null;
}
return getTextFromObject(descriptionObject);
} catch (Exception e) {
throw new ParsingException("Could not get description", e);
}
......
......@@ -79,7 +79,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
InfoItemsSearchCollector collector = getInfoItemSearchCollector();
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents");
......@@ -103,7 +103,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
}
InfoItemsSearchCollector collector = getInfoItemSearchCollector();
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
......
package org.schabi.newpipe.extractor.services;
@SuppressWarnings("unused")
public interface BaseSearchExtractorTest extends BaseListExtractorTest {
void testSearchString() throws Exception;
void testSearchSuggestion() throws Exception;
}
package org.schabi.newpipe.extractor.services;
import org.junit.Test;
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.StreamingService;
import static org.hamcrest.CoreMatchers.*;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
public abstract class DefaultExtractorTest<T extends Extractor> implements BaseExtractorTest {
public abstract T extractor() throws Exception;
public abstract StreamingService expectedService() throws Exception;
public abstract String expectedName() throws Exception;
public abstract String expectedId() throws Exception;
public abstract String expectedUrlContains() throws Exception;
public abstract String expectedOriginalUrlContains() throws Exception;
@Test
@Override
public void testServiceId() throws Exception {
assertEquals(expectedService().getServiceId(), extractor().getServiceId());
}
@Test
@Override
public void testName() throws Exception {
assertEquals(expectedName(), extractor().getName());
}
@Test
@Override
public void testId() throws Exception {
assertEquals(expectedId(), extractor().getId());
}
@Test
@Override
public void testUrl() throws Exception {
final String url = extractor().getUrl();
assertIsSecureUrl(url);
assertThat(url, containsString(expectedUrlContains()));
}
@Test
@Override
public void testOriginalUrl() throws Exception {
final String originalUrl = extractor().getOriginalUrl();
assertIsSecureUrl(originalUrl);
assertThat(originalUrl, containsString(expectedOriginalUrlContains()));
}
}
package org.schabi.newpipe.extractor.services;
import org.junit.Test;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
public abstract class DefaultListExtractorTest<T extends ListExtractor<? extends InfoItem>> extends DefaultExtractorTest<T>
implements BaseListExtractorTest {
@Nullable
public InfoItem.InfoType expectedInfoItemType() {
return null;
}
public boolean expectedHasMoreItems() {
return true;
}
@Test
@Override
public void testRelatedItems() throws Exception {
final ListExtractor<? extends InfoItem> extractor = extractor();
final InfoItem.InfoType expectedType = expectedInfoItemType();
final ListExtractor.InfoItemsPage<? extends InfoItem> items = defaultTestRelatedItems(extractor);
if (expectedType != null) {
assertOnlyContainsType(items, expectedType);
}
}
@Test
@Override
public void testMoreRelatedItems() throws Exception {
final ListExtractor<? extends InfoItem> extractor = extractor();
if (expectedHasMoreItems()) {
final InfoItem.InfoType expectedType = expectedInfoItemType();
final ListExtractor.InfoItemsPage<? extends InfoItem> items = defaultTestMoreItems(extractor);
if (expectedType != null) {
assertOnlyContainsType(items, expectedType);
}
} else {
assertNoMoreItems(extractor);
}
}
}
package org.schabi.newpipe.extractor.services;
import org.junit.Test;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import javax.annotation.Nullable;
import static org.junit.Assert.assertEquals;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTest<SearchExtractor>
implements BaseSearchExtractorTest {
public abstract String expectedSearchString();
@Nullable public abstract String expectedSearchSuggestion();
@Test
@Override
public void testSearchString() throws Exception {
assertEquals(expectedSearchString(), extractor().getSearchString());
}
@Test
@Override
public void testSearchSuggestion() throws Exception {
final String expectedSearchSuggestion = expectedSearchSuggestion();
if (expectedSearchSuggestion == null || expectedSearchSuggestion.isEmpty()) {
assertEmpty("Suggestion was expected to be empty", extractor().getSearchSuggestion());
} else {
assertEquals(expectedSearchSuggestion, extractor().getSearchSuggestion());
}
}
}
......@@ -2,23 +2,22 @@ package org.schabi.newpipe.extractor.services;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.util.Calendar;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.*;
import static org.schabi.newpipe.extractor.StreamingService.*;
import static org.schabi.newpipe.extractor.StreamingService.LinkType;
public final class DefaultTests {
public static void defaultTestListOfItems(StreamingService expectedService, List<? extends InfoItem> itemsList, List<Throwable> errors) throws ParsingException {
......@@ -71,12 +70,38 @@ public final class DefaultTests {
expectedLinkType, linkTypeByUrl);
}
public static void assertOnlyContainsType(ListExtractor.InfoItemsPage<? extends InfoItem> items, InfoItem.InfoType expectedType) {
for (InfoItem item : items.getItems()) {
assertEquals("Item list contains unexpected info types",
expectedType, item.getInfoType());
}
}
public static <T extends InfoItem> void assertNoMoreItems(ListExtractor<T> extractor) throws Exception {
assertFalse("More items available when it shouldn't", extractor.hasNextPage());
final String nextPageUrl = extractor.getNextPageUrl();
assertTrue("Next page is not empty or null", nextPageUrl == null || nextPageUrl.isEmpty());
}
public static void assertNoDuplicatedItems(StreamingService expectedService,
ListExtractor.InfoItemsPage<InfoItem> page1,
ListExtractor.InfoItemsPage<InfoItem> page2) throws Exception {
defaultTestListOfItems(expectedService, page1.getItems(), page1.getErrors());
defaultTestListOfItems(expectedService, page2.getItems(), page2.getErrors());
final Set<String> urlsSet = new HashSet<>();
for (InfoItem item : page1.getItems()) {
urlsSet.add(item.getUrl());
}
for (InfoItem item : page2.getItems()) {
final boolean wasAdded = urlsSet.add(item.getUrl());
if (!wasAdded) {
fail("Same item was on the first and second page item list");
}
}
}
public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestRelatedItems(ListExtractor<T> extractor) throws Exception {
final ListExtractor.InfoItemsPage<T> page = extractor.getInitialPage();
final List<T> itemsList = page.getItems();
......
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.util.Arrays;
import static junit.framework.TestCase.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
/**
* Test for {@link MediaCCCSearchExtractor}
*/
public class MediaCCCSearchExtractorAllTest {
private static SearchExtractor extractor;
private static ListExtractor.InfoItemsPage<InfoItem> itemsPage;
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory()
.fromQuery("c3", Arrays.asList(new String[0]), ""));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}
@Test
public void testIfChannelInfoItemsAvailable() {
boolean isAvialable = false;
for (InfoItem item : itemsPage.getItems()) {
if (item instanceof ChannelInfoItem) {
isAvialable = true;
}
}
assertTrue("ChannelInfoItem not in all list", isAvialable);
}
@Test
public void testIfStreamInfoitemsAvailable() {
boolean isAvialable = false;
for (InfoItem item : itemsPage.getItems()) {
if (item instanceof StreamInfoItem) {
isAvialable = true;
}
}
assertTrue("ChannelInfoItem not in all list", isAvialable);
}
}
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import java.util.Arrays;
import static junit.framework.TestCase.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
/**
* Test for {@link MediaCCCSearchExtractor}
*/
public class MediaCCCSearchExtractorConferencesTest {
private static SearchExtractor extractor;
private static ListExtractor.InfoItemsPage<InfoItem> itemsPage;
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory()
.fromQuery("c3", Arrays.asList(new String[]{"conferences"}), ""));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}
@Test
public void testReturnTypeChannel() {
for (InfoItem item : itemsPage.getItems()) {
assertTrue("Item is not of type channel", item instanceof ChannelInfoItem);
}
}
@Test
public void testItemCount() {
assertTrue("Count is to hight: " + itemsPage.getItems().size(), itemsPage.getItems().size() < 127);
assertTrue("Countis to low: " + itemsPage.getItems().size(), itemsPage.getItems().size() >= 29);
}
}
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.util.Arrays;
import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
/**
* Test for {@link MediaCCCSearchExtractor}
*/
public class MediaCCCSearchExtractorEventsTest {
private static SearchExtractor extractor;
private static ListExtractor.InfoItemsPage<InfoItem> itemsPage;
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory()
.fromQuery("linux", Arrays.asList(new String[]{"events"}), ""));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}
@Test
public void testCount() throws Exception {
assertTrue(Integer.toString(itemsPage.getItems().size()),
itemsPage.getItems().size() >= 25);
}
@Test
public void testServiceId() throws Exception {
assertEquals(2, extractor.getServiceId());
}
@Test
public void testName() throws Exception {
assertFalse(itemsPage.getItems().get(0).getName(), itemsPage.getItems().get(0).getName().isEmpty());
}
@Test
public void testUrl() throws Exception {
assertTrue("Url should start with: https://api.media.ccc.de/public/events/",
itemsPage.getItems().get(0).getUrl().startsWith("https://api.media.ccc.de/public/events/"));
}
@Test
public void testThumbnailUrl() throws