Commit 9005105e authored by Christian Schabesberger's avatar Christian Schabesberger
Browse files

initial commit

parents
package org.schabi.newpipe.extractor;
/**
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* AbstractStreamInfo.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**Common properties between StreamInfo and StreamInfoItem.*/
public abstract class AbstractStreamInfo {
public enum StreamType {
NONE, // placeholder to check if stream type was checked or not
VIDEO_STREAM,
AUDIO_STREAM,
LIVE_STREAM,
AUDIO_LIVE_STREAM,
FILE
}
public StreamType stream_type;
public int service_id = -1;
public String id = "";
public String title = "";
public String uploader = "";
public String thumbnail_url = "";
public String webpage_url = "";
public String upload_date = "";
public long view_count = -1;
}
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream_info.AudioStream;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Vector;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
/**
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* DashMpdParser.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class DashMpdParser {
private DashMpdParser() {
}
static class DashMpdParsingException extends ParsingException {
DashMpdParsingException(String message, Exception e) {
super(message, e);
}
}
public static List<AudioStream> getAudioStreams(String dashManifestUrl)
throws DashMpdParsingException, ReCaptchaException {
String dashDoc;
Downloader downloader = NewPipe.getDownloader();
try {
dashDoc = downloader.download(dashManifestUrl);
} catch(IOException ioe) {
throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge needed");
}
Vector<AudioStream> audioStreams = new Vector<>();
try {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder = factory.newDocumentBuilder();
InputStream stream = new ByteArrayInputStream(dashDoc.getBytes());
Document doc = builder.parse(stream);
NodeList adaptationSetList = doc.getElementsByTagName("AdaptationSet");
for(int i = 0; i < adaptationSetList.getLength(); i++) {
Element adaptationSet = (Element) adaptationSetList.item(i);
String memeType = adaptationSet.getAttribute("mimeType");
if(memeType.contains("audio")) {
Element representation = (Element) adaptationSet.getElementsByTagName("Representation").item(0);
String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent();
int bandwidth = Integer.parseInt(representation.getAttribute("bandwidth"));
int samplingRate = Integer.parseInt(representation.getAttribute("audioSamplingRate"));
int format = -1;
if(memeType.equals(MediaFormat.WEBMA.mimeType)) {
format = MediaFormat.WEBMA.id;
} else if(memeType.equals(MediaFormat.M4A.mimeType)) {
format = MediaFormat.M4A.id;
}
audioStreams.add(new AudioStream(url, format, bandwidth, samplingRate));
}
}
}
catch(Exception e) {
throw new DashMpdParsingException("Could not parse Dash mpd", e);
}
return audioStreams;
}
}
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import java.io.IOException;
import java.util.Map;
/**
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* Downloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface Downloader {
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl, String language) throws IOException, ReCaptchaException;
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException;
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl) throws IOException, ReCaptchaException;
}
package org.schabi.newpipe.extractor;
/**
* Created by the-scrabi on 11.02.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* InfoItem.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface InfoItem {
public enum InfoType {
STREAM,
PLAYLIST,
CHANNEL
}
InfoType infoType();
String getTitle();
String getLink();
}
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.util.List;
import java.util.Vector;
/**
* Created by Christian Schabesberger on 12.02.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* InfoItemCollector.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class InfoItemCollector {
private List<InfoItem> itemList = new Vector<>();
private List<Throwable> errors = new Vector<>();
private int serviceId = -1;
public InfoItemCollector(int serviceId) {
this.serviceId = serviceId;
}
public List<InfoItem> getItemList() {
return itemList;
}
public List<Throwable> getErrors() {
return errors;
}
protected void addFromCollector(InfoItemCollector otherC) throws ExtractionException {
if(serviceId != otherC.serviceId) {
throw new ExtractionException("Service Id does not equal: "
+ NewPipe.getNameOfService(serviceId)
+ " and " + NewPipe.getNameOfService(otherC.serviceId));
}
errors.addAll(otherC.errors);
itemList.addAll(otherC.itemList);
}
protected void addError(Exception e) {
errors.add(e);
}
protected void addItem(InfoItem item) {
itemList.add(item);
}
protected int getServiceId() {
return serviceId;
}
}
This diff is collapsed.
package org.schabi.newpipe.extractor;
/**
* Created by Adam Howard on 08/11/15.
*
* Copyright (c) Christian Schabesberger <chris.schabesberger@mailbox.org>
* and Adam Howard <achdisposable1@gmail.com> 2015
*
* MediaFormat.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**Static data about various media formats support by Newpipe, eg mime type, extension*/
public enum MediaFormat {
//video and audio combined formats
// id name suffix mime type
MPEG_4 (0x0, "MPEG-4", "mp4", "video/mp4"),
v3GPP (0x1, "3GPP", "3gp", "video/3gpp"),
WEBM (0x2, "WebM", "webm", "video/webm"),
// audio formats
M4A (0x3, "m4a", "m4a", "audio/mp4"),
WEBMA (0x4, "WebM", "webm", "audio/webm");
public final int id;
@SuppressWarnings("WeakerAccess")
public final String name;
@SuppressWarnings("WeakerAccess")
public final String suffix;
public final String mimeType;
MediaFormat(int id, String name, String suffix, String mimeType) {
this.id = id;
this.name = name;
this.suffix = suffix;
this.mimeType = mimeType;
}
/**Return the friendly name of the media format with the supplied id
* @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number.
* @return the friendly name of the MediaFormat associated with this ids,
* or an empty String if none match it.*/
public static String getNameById(int ident) {
for (MediaFormat vf : MediaFormat.values()) {
if(vf.id == ident) return vf.name;
}
return "";
}
/**Return the file extension of the media format with the supplied id
* @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number.
* @return the file extension of the MediaFormat associated with this ids,
* or an empty String if none match it.*/
public static String getSuffixById(int ident) {
for (MediaFormat vf : MediaFormat.values()) {
if(vf.id == ident) return vf.suffix;
}
return "";
}
/**Return the MIME type of the media format with the supplied id
* @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number.
* @return the MIME type of the MediaFormat associated with this ids,
* or an empty String if none match it.*/
public static String getMimeById(int ident) {
for (MediaFormat vf : MediaFormat.values()) {
if(vf.id == ident) return vf.mimeType;
}
return "";
}
}
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
/**
* Created by Christian Schabesberger on 23.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* NewPipe.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**Provides access to the video streaming services supported by NewPipe.
* Currently only Youtube until the API becomes more stable.*/
@SuppressWarnings("ALL")
public class NewPipe {
private NewPipe() {
}
private static final String TAG = NewPipe.class.toString();
private static Downloader downloader = null;
public static StreamingService[] getServices() {
return ServiceList.serviceList;
}
public static StreamingService getService(int serviceId)throws ExtractionException {
for(StreamingService s : ServiceList.serviceList) {
if(s.getServiceId() == serviceId) {
return s;
}
}
return null;
}
public static StreamingService getService(String serviceName) throws ExtractionException {
return ServiceList.serviceList[getIdOfService(serviceName)];
}
public static String getNameOfService(int id) {
try {
return getService(id).getServiceInfo().name;
} catch (Exception e) {
System.err.println("Service id not known");
e.printStackTrace();
return "";
}
}
public static int getIdOfService(String serviceName) {
for(int i = 0; i < ServiceList.serviceList.length; i++) {
if(ServiceList.serviceList[i].getServiceInfo().name.equals(serviceName)) {
return i;
}
}
return -1;
}
public static void init(Downloader d) {
downloader = d;
}
public static Downloader getDownloader() {
return downloader;
}
public static StreamingService getServiceByUrl(String url) {
for(StreamingService s : ServiceList.serviceList) {
if(s.getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) {
return s;
}
}
return null;
}
}
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* Parser.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/** avoid using regex !!! */
public class Parser {
private Parser() {
}
public static class RegexException extends ParsingException {
public RegexException(String message) {
super(message);
}
}
public static String matchGroup1(String pattern, String input) throws RegexException {
return matchGroup(pattern, input, 1);
}
public static String matchGroup(String pattern, String input, int group) throws RegexException {
Pattern pat = Pattern.compile(pattern);
Matcher mat = pat.matcher(input);
boolean foundMatch = mat.find();
if (foundMatch) {
return mat.group(group);
}
else {
//Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\"");
if(input.length() > 1024) {
throw new RegexException("failed to find pattern \""+pattern);
} else {
throw new RegexException("failed to find pattern \"" + pattern + " inside of " + input + "\"");
}
}
}
public static Map<String, String> compatParseMap(final String input) throws UnsupportedEncodingException {
Map<String, String> map = new HashMap<>();
for(String arg : input.split("&")) {
String[] splitArg = arg.split("=");
if(splitArg.length > 1) {
map.put(splitArg[0], URLDecoder.decode(splitArg[1], "UTF-8"));
} else {
map.put(splitArg[0], "");
}
}
return map;
}
}
NewPipe Extractor
=================
This is the system independent core of [NewPipe](https://github.com/TeamNewPipe/NewPipe).
It can be used to create your own java based NewPipe client.
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
/**
* Created by the-scrabi on 18.02.17.
*/
class ServiceList {
public static final StreamingService[] serviceList = {
new YoutubeService(0)
};
}
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.stream_info.StreamExtractor;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 23.08.15.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* StreamingService.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public abstract class StreamingService {
public class ServiceInfo {
public String name = "";
}
public enum LinkType {
NONE,
STREAM,
CHANNEL,
PLAYLIST
}
private int serviceId;
public StreamingService(int id) {
serviceId = id;
}
public abstract ServiceInfo getServiceInfo();
public abstract StreamExtractor getExtractorInstance(String url)
throws IOException, ExtractionException;
public abstract SearchEngine getSearchEngineInstance();
public abstract UrlIdHandler getStreamUrlIdHandlerInstance();
public abstract UrlIdHandler getChannelUrlIdHandlerInstance();
public abstract ChannelExtractor getChannelExtractorInstance(String url, int page)
throws ExtractionException, IOException;
public abstract SuggestionExtractor getSuggestionExtractorInstance();
public final int getServiceId() {
return serviceId;
}
/**
* figure out where the link is pointing to (a channel, video, playlist, etc.)
*/