Commit 22fa1319 authored by TobiGr's avatar TobiGr
Browse files

Merge branch 'dev' into bandcamp

parents a1688fe9 cb07ffa1
......@@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo.
If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps:
1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.10'`the `dependencies` in your `build.gradle`. Replace `v0.20.10` with the latest release.
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.11'`the `dependencies` in your `build.gradle`. Replace `v0.20.11` with the latest release.
**Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required.
......
......@@ -2,10 +2,13 @@ allprojects {
apply plugin: 'java-library'
apply plugin: 'maven'
compileJava.options.encoding = 'UTF-8'
compileTestJava.options.encoding = 'UTF-8'
sourceCompatibility = 1.8
targetCompatibility = 1.8
version 'v0.20.10'
version 'v0.20.11'
group 'com.github.TeamNewPipe'
repositories {
......
......@@ -40,7 +40,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class SoundcloudParsingHelper {
private static final String HARDCODED_CLIENT_ID = "H2c34Q0E7hftqnuDHGsk88DbNqhYpgMm"; // Updated on 24/06/20
private static final String HARDCODED_CLIENT_ID = "Kl9G8jQT22DxqatQk09IjWRujGlut5Vd"; // Updated on 04/03/21
private static String clientId;
private SoundcloudParsingHelper() {
......@@ -67,7 +67,7 @@ public class SoundcloudParsingHelper {
Collections.reverse(possibleScripts);
final HashMap<String, List<String>> headers = new HashMap<>();
headers.put("Range", singletonList("bytes=0-16384"));
headers.put("Range", singletonList("bytes=0-50000"));
for (Element element : possibleScripts) {
final String srcUrl = element.attr("src");
......
package org.schabi.newpipe.extractor.services.youtube;
import com.grack.nanojson.*;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response;
......@@ -10,6 +15,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
......@@ -33,7 +39,12 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.Utils.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.join;
/*
* Created by Christian Schabesberger on 02.03.16.
......@@ -638,7 +649,7 @@ public class YoutubeParsingHelper {
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(url, headers, localization);
return toJsonArray(getValidJsonResponseBody(response));
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
public static JsonArray getJsonResponse(final Page page, final Localization localization)
......@@ -652,15 +663,7 @@ public class YoutubeParsingHelper {
final Response response = getDownloader().get(page.getUrl(), headers, localization);
return toJsonArray(getValidJsonResponseBody(response));
}
public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
try {
return JsonParser.array().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
/**
......@@ -821,4 +824,14 @@ public class YoutubeParsingHelper {
return false;
}
public static String unescapeDocument(final String doc) {
return doc
.replaceAll("\\\\x22", "\"")
.replaceAll("\\\\x7b", "{")
.replaceAll("\\\\x7d", "}")
.replaceAll("\\\\x5b", "[")
.replaceAll("\\\\x5d", "]");
}
}
......@@ -2,10 +2,13 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
......@@ -15,13 +18,20 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
......@@ -226,7 +236,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
......@@ -254,27 +264,44 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
// as they don't deliver enough information on their own (the channel name, for example).
fetchPage();
// @formatter:off
byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("clientName", "1")
.value("clientVersion", getClientVersion())
.end()
.end()
.value("continuation", page.getId())
.end()
.done()
.getBytes(UTF_8);
// @formatter:on
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getArray("onResponseReceivedActions").getObject(0).getObject("appendContinuationItemsAction");
JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction");
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation.getArray("continuationItems"));
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
}
private Page getNextPageFrom(final JsonObject continuations) {
private Page getNextPageFrom(final JsonObject continuations) throws IOException, ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand").getString("token");
final String clickTrackingParams = continuationEndpoint.getString("clickTrackingParams");
return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation
+ "&continuation=" + continuation + "&itct=" + clickTrackingParams);
return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
continuation);
}
/**
......
......@@ -15,6 +15,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
......@@ -46,7 +47,10 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
final String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
String commentsTokenInside = findValue(responseBody, "sectionListRenderer", "}");
if (!commentsTokenInside.contains("continuation\":\"")) {
commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
}
final String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
return getPage(getNextPage(commentsToken));
}
......@@ -128,7 +132,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
final Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("User-Agent", singletonList(USER_AGENT));
final Response response = downloader.get(getUrl(), requestHeaders, getExtractorLocalization());
responseBody = response.responseBody();
responseBody = YoutubeParsingHelper.unescapeDocument(response.responseBody());
ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\"");
ytClientName = Parser.matchGroup1(YT_CLIENT_NAME_PATTERN, responseBody);
}
......@@ -158,16 +162,9 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
return result.toString();
}
private String findValue(String doc, String start, String end) {
final String unescaped = doc
.replaceAll("\\\\x22", "\"")
.replaceAll("\\\\x7b", "{")
.replaceAll("\\\\x7d", "}")
.replaceAll("\\\\x5b", "[")
.replaceAll("\\\\x5d", "]");
final int beginIndex = unescaped.indexOf(start) + start.length();
final int endIndex = unescaped.indexOf(end, beginIndex);
return unescaped.substring(beginIndex, endIndex);
private String findValue(final String doc, final String start, final String end) {
final int beginIndex = doc.indexOf(start) + start.length();
final int endIndex = doc.indexOf(end, beginIndex);
return doc.substring(beginIndex, endIndex);
}
}
......@@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
......@@ -14,14 +15,19 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
......@@ -51,7 +57,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
final Response response = getResponse(url, getExtractorLocalization());
final JsonArray ajaxJson = toJsonArray(response.responseBody());
final JsonArray ajaxJson = JsonUtils.toJsonArray(response.responseBody());
initialData = ajaxJson.getObject(3).getObject("response");
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("playlist").getObject("playlist");
......
......@@ -2,9 +2,12 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
......@@ -19,11 +22,20 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.utils.JsonUtils.toJsonObject;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@SuppressWarnings("WeakerAccess")
......@@ -168,7 +180,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
......@@ -205,12 +217,27 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
// @formatter:off
byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("clientName", "1")
.value("clientVersion", getClientVersion())
.end()
.end()
.value("continuation", page.getId())
.end()
.done()
.getBytes(UTF_8);
// @formatter:on
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
final JsonObject ajaxJson = toJsonObject(getValidJsonResponseBody(response));
final JsonArray continuation = ajaxJson.getObject(1)
.getObject("response")
.getArray("onResponseReceivedActions")
final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction")
.getArray("continuationItems");
......@@ -220,7 +247,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
}
private Page getNextPageFrom(final JsonArray contents) {
private Page getNextPageFrom(final JsonArray contents) throws IOException, ExtractionException {
if (isNullOrEmpty(contents)) {
return null;
}
......@@ -232,7 +259,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.getObject("continuationEndpoint")
.getObject("continuationCommand")
.getString("token");
return new Page("https://www.youtube.com/browse_ajax?continuation=" + continuation);
return new Page(
"https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
continuation);
} else {
return null;
}
......
......@@ -4,16 +4,18 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public class JsonUtils {
public static final JsonObject EMPTY_OBJECT = new JsonObject();
public static final JsonArray EMPTY_ARRAY = new JsonArray();
......@@ -103,6 +105,22 @@ public class JsonUtils {
return result;
}
public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
try {
return JsonParser.array().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
}
public static JsonObject toJsonObject(final String responseBody) throws ParsingException {
try {
return JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
}
/**
* <p>Get an attribute of a web page as JSON
*
......
......@@ -13,7 +13,11 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.*;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.ContinuationsTests;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.HugePlaylist;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.LearningPlaylist;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.NotAvailable;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.TimelessPopHits;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
......@@ -410,7 +414,7 @@ public class YoutubePlaylistExtractorTest {
public void testOnlySingleContinuation() throws Exception {
final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/playlist?list=PLjgwFL8urN2DFRuRkFTkmtHjyoNWHHdZX");
"https://www.youtube.com/playlist?list=PLoumn5BIsUDeGF1vy5Nylf_RJKn5aL_nr");
extractor.fetchPage();
final ListExtractor.InfoItemsPage<StreamInfoItem> page = defaultTestMoreItems(
......
......@@ -280,7 +280,8 @@ public class YoutubeSearchExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "verified"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(CHANNELS), "");
extractor.fetchPage();
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment