Unverified Commit 021da75f authored by Tobi's avatar Tobi Committed by GitHub
Browse files

Merge pull request #232 from fynngodau/dev

Bandcamp support
parents cb07ffa1 22fa1319
......@@ -44,6 +44,7 @@ The following sites are currently supported:
- SoundCloud
- media.ccc.de
- PeerTube (no P2P)
- Bandcamp
## License
......
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.services.bandcamp.BandcampService;
import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService;
import org.schabi.newpipe.extractor.services.peertube.PeertubeService;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService;
......@@ -39,6 +40,7 @@ public final class ServiceList {
public static final SoundcloudService SoundCloud;
public static final MediaCCCService MediaCCC;
public static final PeertubeService PeerTube;
public static final BandcampService Bandcamp;
/**
* When creating a new service, put this service in the end of this list,
......@@ -49,7 +51,8 @@ public final class ServiceList {
YouTube = new YoutubeService(0),
SoundCloud = new SoundcloudService(1),
MediaCCC = new MediaCCCService(2),
PeerTube = new PeertubeService(3)
PeerTube = new PeertubeService(3),
Bandcamp = new BandcampService(4)
));
/**
......
......@@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Utils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
......@@ -13,7 +14,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
///////////////////////////////////
public List<String> getContentFilter(String url) throws ParsingException {
return new ArrayList<>(0);
return Collections.emptyList();
}
public String getSortFilter(String url) throws ParsingException {
......
// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later
package org.schabi.newpipe.extractor.services.bandcamp;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.*;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.*;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.util.Collections;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.FEATURED_API_URL;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.KIOSK_FEATURED;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.KIOSK_RADIO;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.RADIO_API_URL;
public class BandcampService extends StreamingService {
public BandcampService(final int id) {
super(id, "Bandcamp", Collections.singletonList(AUDIO));
}
@Override
public String getBaseUrl() {
return BASE_URL;
}
@Override
public LinkHandlerFactory getStreamLHFactory() {
return new BandcampStreamLinkHandlerFactory();
}
@Override
public ListLinkHandlerFactory getChannelLHFactory() {
return new BandcampChannelLinkHandlerFactory();
}
@Override
public ListLinkHandlerFactory getPlaylistLHFactory() {
return new BandcampPlaylistLinkHandlerFactory();
}
@Override
public SearchQueryHandlerFactory getSearchQHFactory() {
return new BandcampSearchQueryHandlerFactory();
}
@Override
public ListLinkHandlerFactory getCommentsLHFactory() {
return null;
}
@Override
public SearchExtractor getSearchExtractor(final SearchQueryHandler queryHandler) {
return new BandcampSearchExtractor(this, queryHandler);
}
@Override
public SuggestionExtractor getSuggestionExtractor() {
return new BandcampSuggestionExtractor(this);
}
@Override
public SubscriptionExtractor getSubscriptionExtractor() {
return null;
}
@Override
public KioskList getKioskList() throws ExtractionException {
KioskList kioskList = new KioskList(this);
try {
kioskList.addKioskEntry((streamingService, url, kioskId) ->
new BandcampFeaturedExtractor(
BandcampService.this,
new BandcampFeaturedLinkHandlerFactory().fromUrl(FEATURED_API_URL), kioskId),
new BandcampFeaturedLinkHandlerFactory(), KIOSK_FEATURED);
kioskList.addKioskEntry((streamingService, url, kioskId) ->
new BandcampRadioExtractor(BandcampService.this,
new BandcampFeaturedLinkHandlerFactory().fromUrl(RADIO_API_URL), kioskId),
new BandcampFeaturedLinkHandlerFactory(), KIOSK_RADIO);
kioskList.setDefaultKiosk(KIOSK_FEATURED);
} catch (final Exception e) {
throw new ExtractionException(e);
}
return kioskList;
}
@Override
public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) {
return new BandcampChannelExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return new BandcampPlaylistExtractor(this, linkHandler);
}
@Override
public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) {
if (BandcampExtractorHelper.isRadioUrl(linkHandler.getUrl()))
return new BandcampRadioStreamExtractor(this, linkHandler);
else
return new BandcampStreamExtractor(this, linkHandler);
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) {
return null;
}
}
// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.jsoup.Jsoup;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampDiscographStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
public class BandcampChannelExtractor extends ChannelExtractor {
private JsonObject channelInfo;
public BandcampChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public String getAvatarUrl() {
if (channelInfo.getLong("bio_image_id") == 0) return "";
return BandcampExtractorHelper.getImageUrl(channelInfo.getLong("bio_image_id"), false);
}
@Override
public String getBannerUrl() throws ParsingException {
/*
* Why does the mobile endpoint not contain the header?? Or at least not the same one?
* Anyway we're back to querying websites
*/
try {
final String html = getDownloader()
.get(channelInfo.getString("bandcamp_url").replace("http://", "https://"))
.responseBody();
return Jsoup.parse(html)
.getElementById("customHeader")
.getElementsByTag("img")
.first()
.attr("src");
} catch (final IOException | ReCaptchaException e) {
throw new ParsingException("Could not download artist web site", e);
} catch (final NullPointerException e) {
// No banner available
return "";
}
}
/**
* bandcamp stopped providing RSS feeds when appending /feed to any URL
* because too few people used it.
*/
@Override
public String getFeedUrl() {
return null;
}
@Override
public long getSubscriberCount() {
return -1;
}
@Override
public String getDescription() {
return channelInfo.getString("bio");
}
@Override
public String getParentChannelName() {
return null;
}
@Override
public String getParentChannelUrl() {
return null;
}
@Override
public String getParentChannelAvatarUrl() {
return null;
}
@Override
public boolean isVerified() throws ParsingException {
return false;
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray discography = channelInfo.getArray("discography");
for (int i = 0; i < discography.size(); i++) {
// I define discograph as an item that can appear in a discography
final JsonObject discograph = discography.getObject(i);
if (!discograph.getString("item_type").equals("track")) continue;
collector.commit(new BandcampDiscographStreamInfoItemExtractor(discograph, getUrl()));
}
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(Page page) {
return null;
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
channelInfo = BandcampExtractorHelper.getArtistDetails(getId());
}
@Nonnull
@Override
public String getName() {
return channelInfo.getString("name");
}
}
// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class BandcampChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private final Element resultInfo, searchResult;
public BandcampChannelInfoItemExtractor(final Element searchResult) {
this.searchResult = searchResult;
resultInfo = searchResult.getElementsByClass("result-info").first();
}
@Override
public String getName() throws ParsingException {
return resultInfo.getElementsByClass("heading").text();
}
@Override
public String getUrl() throws ParsingException {
return resultInfo.getElementsByClass("itemurl").text();
}
@Override
public String getThumbnailUrl() throws ParsingException {
final Element img = searchResult.getElementsByClass("art").first()
.getElementsByTag("img").first();
if (img != null) {
return img.attr("src");
} else {
return null;
}
}
@Override
public String getDescription() {
return resultInfo.getElementsByClass("subhead").text();
}
@Override
public long getSubscriberCount() {
return -1;
}
@Override
public long getStreamCount() {
return -1;
}
@Override
public boolean isVerified() throws ParsingException {
return false;
}
}
// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.time.DateTimeException;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
public class BandcampExtractorHelper {
public static final String BASE_URL = "https://bandcamp.com";
public static final String BASE_API_URL = BASE_URL + "/api";
/**
* Translate all these parameters together to the URL of the corresponding album or track
* using the mobile API
*/
public static String getStreamUrlFromIds(final long bandId, final long itemId, final String itemType)
throws ParsingException {
try {
final String jsonString = NewPipe.getDownloader().get(
BASE_API_URL + "/mobile/22/tralbum_details?band_id=" + bandId
+ "&tralbum_id=" + itemId + "&tralbum_type=" + itemType.charAt(0))
.responseBody();
return JsonParser.object().from(jsonString)
.getString("bandcamp_url").replace("http://", "https://");
} catch (final JsonParserException | ReCaptchaException | IOException e) {
throw new ParsingException("Ids could not be translated to URL", e);
}
}
/**
* Fetch artist details from mobile endpoint.
* <a href=https://notabug.org/fynngodau/bandcampDirect/wiki/rewindBandcamp+%E2%80%93+Fetching+artist+details>
* More technical info.</a>
*/
public static JsonObject getArtistDetails(String id) throws ParsingException {
try {
return
JsonParser.object().from(
NewPipe.getDownloader().post(
BASE_API_URL + "/mobile/22/band_details",
null,
JsonWriter.string()
.object()
.value("band_id", id)
.end()
.done()
.getBytes()
).responseBody()
);
} catch (final IOException | ReCaptchaException | JsonParserException e) {
throw new ParsingException("Could not download band details", e);
}
}
/**
* @param id The image ID
* @param album Whether this is the cover of an album
* @return URL of image with this ID in size 10 which is 1200x1200 (we could also choose size 0
* but we don't want something as large as 3460x3460 here)
*/
public static String getImageUrl(final long id, final boolean album) {
return "https://f4.bcbits.com/img/" + (album ? 'a' : "") + id + "_10.jpg";
}
/**
* @return <code>true</code> if the given URL looks like it comes from a bandcamp custom domain
* or if it comes from <code>bandcamp.com</code> itself
*/
public static boolean isSupportedDomain(final String url) throws ParsingException {
// Accept all bandcamp.com URLs
if (url.toLowerCase().matches("https?://.+\\.bandcamp\\.com(/.*)?")) return true;
try {
// Accept all other URLs if they contain a <meta> tag that says they are generated by bandcamp
return Jsoup.parse(
NewPipe.getDownloader().get(url).responseBody()
)
.getElementsByAttributeValue("name", "generator")
.attr("content").equals("Bandcamp");
} catch (IOException | ReCaptchaException e) {
throw new ParsingException("Could not determine whether URL is custom domain " +
"(not available? network error?)");
}
}
/**
* Whether the URL points to a radio kiosk.
* @param url the URL to check
* @return true if the URL matches <code>https://bandcamp.com/?show=SHOW_ID</code>
*/
public static boolean isRadioUrl(final String url) {
return url.toLowerCase().matches("https?://bandcamp\\.com/\\?show=\\d+");
}
static DateWrapper parseDate(final String textDate) throws ParsingException {
try {
final ZonedDateTime zonedDateTime = ZonedDateTime.parse(
textDate, DateTimeFormatter.ofPattern("dd MMM yyyy HH:mm:ss zzz", Locale.ENGLISH));
return new DateWrapper(zonedDateTime.toOffsetDateTime(), false);
} catch (final DateTimeException e) {
throw new ParsingException("Could not parse date '" + textDate + "'", e);
}
}
}
// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL;
public class BandcampFeaturedExtractor extends KioskExtractor<PlaylistInfoItem> {
public static final String KIOSK_FEATURED = "Featured";
public static final String FEATURED_API_URL = BASE_API_URL + "/mobile/24/bootstrap_data";
private JsonObject json;
public BandcampFeaturedExtractor(final StreamingService streamingService, final ListLinkHandler listLinkHandler,
final String kioskId) {
super(streamingService, listLinkHandler, kioskId);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
try {
json = JsonParser.object().from(
getDownloader().post(
FEATURED_API_URL, null, "{\"platform\":\"\",\"version\":0}".getBytes()
).responseBody()
);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse Bandcamp featured API response", e);
}
}
@Nonnull
@Override
public String getName() throws ParsingException {