Commit 2b671b15 authored by TobiGr's avatar TobiGr
Browse files

[YouTube] Use new continuation API for channels


Co-authored-by: default avatarTiA4f8R <74829229+tia4f8r@users.noreply.github.com>
parent 9256b3b8
......@@ -2,10 +2,13 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
......@@ -15,13 +18,20 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
......@@ -226,7 +236,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
......@@ -254,18 +264,36 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
// as they don't deliver enough information on their own (the channel name, for example).
fetchPage();
// @formatter:off
byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("clientName", "1")
.value("clientVersion", getClientVersion())
.end()
.end()
.value("continuation", page.getId())
.end()
.done()
.getBytes(UTF_8);
// @formatter:on
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getArray("onResponseReceivedActions").getObject(0).getObject("appendContinuationItemsAction");
JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction");
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation.getArray("continuationItems"));
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
}
private Page getNextPageFrom(final JsonObject continuations) {
private Page getNextPageFrom(final JsonObject continuations) throws IOException, ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
......@@ -273,8 +301,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand").getString("token");
final String clickTrackingParams = continuationEndpoint.getString("clickTrackingParams");
return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation
+ "&continuation=" + continuation + "&itct=" + clickTrackingParams);
return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
continuation);
}
/**
......
......@@ -20,7 +20,7 @@
},
"response": {
"responseCode": 404,
"responseMessage": "",
"responseMessage": "Not Found",
"responseHeaders": {
"alt-svc": [
"h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
......@@ -32,7 +32,7 @@
"text/html; charset\u003dutf-8"
],
"date": [
"Sat, 13 Feb 2021 19:13:39 GMT"
"Thu, 04 Mar 2021 14:15:28 GMT"
],
"expires": [
"Mon, 01 Jan 1990 00:00:00 GMT"
......@@ -47,12 +47,15 @@
"ESF"
],
"set-cookie": [
"YSC\u003dbFLZyqvSsDQ; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003dYgNWpn3F6Ww; Domain\u003d.youtube.com; Expires\u003dThu, 12-Aug-2021 19:13:39 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
"YSC\u003dOHweFRBSiuo; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003dDjGaxgk-6jk; Domain\u003d.youtube.com; Expires\u003dTue, 31-Aug-2021 14:15:28 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"transfer-encoding": [
"chunked"
],
"x-content-type-options": [
"nosniff"
],
......@@ -63,7 +66,7 @@
"0"
]
},
"responseBody": "\u003chtml lang\u003d\"en-GB\" dir\u003d\"ltr\"\u003e\u003chead\u003e\u003ctitle\u003e404 Not Found\u003c/title\u003e\u003cstyle nonce\u003d\"AkiRKVBSwx7zy61zR/BBpg\"\u003e*{margin:0;padding:0;border:0}html,body{height:100%;}\u003c/style\u003e\u003clink rel\u003d\"shortcut icon\" href\u003d\"https://www.youtube.com/img/favicon.ico\" type\u003d\"image/x-icon\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_32.png\" sizes\u003d\"32x32\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_48.png\" sizes\u003d\"48x48\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_96.png\" sizes\u003d\"96x96\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_144.png\" sizes\u003d\"144x144\"\u003e\u003c/head\u003e\u003cbody\u003e\u003ciframe style\u003d\"display:block;border:0;\" src\u003d\"/error?src\u003d404\u0026amp;ifr\u003d1\u0026amp;error\u003d\" width\u003d\"100%\" height\u003d\"100%\" frameborder\u003d\"\\\" scrolling\u003d\"no\"\u003e\u003c/iframe\u003e\u003c/body\u003e\u003c/html\u003e",
"responseBody": "\u003chtml lang\u003d\"en-GB\" dir\u003d\"ltr\"\u003e\u003chead\u003e\u003ctitle\u003e404 Not Found\u003c/title\u003e\u003cstyle nonce\u003d\"H7HK+DtuIlOkI+rlUnUlnQ\"\u003e*{margin:0;padding:0;border:0}html,body{height:100%;}\u003c/style\u003e\u003clink rel\u003d\"shortcut icon\" href\u003d\"https://www.youtube.com/img/favicon.ico\" type\u003d\"image/x-icon\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_32.png\" sizes\u003d\"32x32\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_48.png\" sizes\u003d\"48x48\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_96.png\" sizes\u003d\"96x96\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_144.png\" sizes\u003d\"144x144\"\u003e\u003c/head\u003e\u003cbody\u003e\u003ciframe style\u003d\"display:block;border:0;\" src\u003d\"/error?src\u003d404\u0026amp;ifr\u003d1\u0026amp;error\u003d\" width\u003d\"100%\" height\u003d\"100%\" frameborder\u003d\"\\\" scrolling\u003d\"no\"\u003e\u003c/iframe\u003e\u003c/body\u003e\u003c/html\u003e",
"latestUrl": "https://www.youtube.com/channel/DOESNT-EXIST/videos?pbj\u003d1\u0026view\u003d0\u0026flow\u003dgrid"
}
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment