initial commit

AbstractStreamInfo.java

+package org.schabi.newpipe.extractor;
+
+/**
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * AbstractStreamInfo.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**Common properties between StreamInfo and StreamInfoItem.*/
+public abstract class AbstractStreamInfo {
+    public enum StreamType {
+        NONE,   // placeholder to check if stream type was checked or not
+        VIDEO_STREAM,
+        AUDIO_STREAM,
+        LIVE_STREAM,
+        AUDIO_LIVE_STREAM,
+        FILE
+    }
+
+    public StreamType stream_type;
+    public int service_id = -1;
+    public String id = "";
+    public String title = "";
+    public String uploader = "";
+    public String thumbnail_url = "";
+    public String webpage_url = "";
+    public String upload_date = "";
+    public long view_count = -1;
+}

DashMpdParser.java

+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+import org.schabi.newpipe.extractor.stream_info.AudioStream;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Vector;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+/**
+ * Created by Christian Schabesberger on 02.02.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * DashMpdParser.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class DashMpdParser {
+
+    private DashMpdParser() {
+    }
+
+    static class DashMpdParsingException extends ParsingException {
+        DashMpdParsingException(String message, Exception e) {
+            super(message, e);
+        }
+    }
+
+    public static List<AudioStream> getAudioStreams(String dashManifestUrl)
+            throws DashMpdParsingException, ReCaptchaException {
+        String dashDoc;
+        Downloader downloader = NewPipe.getDownloader();
+        try {
+            dashDoc = downloader.download(dashManifestUrl);
+        } catch(IOException ioe) {
+            throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe);
+        } catch (ReCaptchaException e) {
+            throw new ReCaptchaException("reCaptcha Challenge needed");
+        }
+        Vector<AudioStream> audioStreams = new Vector<>();
+
+        try {
+            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+            DocumentBuilder builder = factory.newDocumentBuilder();
+            InputStream stream = new ByteArrayInputStream(dashDoc.getBytes());
+
+            Document doc = builder.parse(stream);
+            NodeList adaptationSetList = doc.getElementsByTagName("AdaptationSet");
+            for(int i = 0; i < adaptationSetList.getLength(); i++) {
+                Element adaptationSet = (Element) adaptationSetList.item(i);
+                String memeType = adaptationSet.getAttribute("mimeType");
+                if(memeType.contains("audio")) {
+                    Element representation = (Element) adaptationSet.getElementsByTagName("Representation").item(0);
+                    String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent();
+                    int bandwidth = Integer.parseInt(representation.getAttribute("bandwidth"));
+                    int samplingRate = Integer.parseInt(representation.getAttribute("audioSamplingRate"));
+                    int format = -1;
+                    if(memeType.equals(MediaFormat.WEBMA.mimeType)) {
+                        format = MediaFormat.WEBMA.id;
+                    } else if(memeType.equals(MediaFormat.M4A.mimeType)) {
+                        format = MediaFormat.M4A.id;
+                    }
+                    audioStreams.add(new AudioStream(url, format, bandwidth, samplingRate));
+                }
+            }
+        }
+        catch(Exception e) {
+            throw new DashMpdParsingException("Could not parse Dash mpd", e);
+        }
+        return audioStreams;
+    }
+}

Downloader.java

+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Created by Christian Schabesberger on 28.01.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * Downloader.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public interface Downloader {
+
+    /**Download the text file at the supplied URL as in download(String),
+     * but set the HTTP header field "Accept-Language" to the supplied string.
+     * @param siteUrl the URL of the text file to return the contents of
+     * @param language the language (usually a 2-character code) to set as the preferred language
+     * @return the contents of the specified text file
+     * @throws IOException*/
+    String download(String siteUrl, String language) throws IOException, ReCaptchaException;
+
+    /**Download the text file at the supplied URL as in download(String),
+     * but set the HTTP header field "Accept-Language" to the supplied string.
+     * @param siteUrl the URL of the text file to return the contents of
+     * @param customProperties set request header properties
+     * @return the contents of the specified text file
+     * @throws IOException*/
+    String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException;
+
+    /**Download (via HTTP) the text file located at the supplied URL, and return its contents.
+     * Primarily intended for downloading web pages.
+     * @param siteUrl the URL of the text file to download
+     * @return the contents of the specified text file
+     * @throws IOException*/
+    String download(String siteUrl) throws IOException, ReCaptchaException;
+}

InfoItem.java

+package org.schabi.newpipe.extractor;
+
+/**
+ * Created by the-scrabi on 11.02.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * InfoItem.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public interface InfoItem {
+    public enum InfoType {
+        STREAM,
+        PLAYLIST,
+        CHANNEL
+    }
+
+    InfoType infoType();
+    String getTitle();
+    String getLink();
+}

InfoItemCollector.java

+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+
+import java.util.List;
+import java.util.Vector;
+
+/**
+ * Created by Christian Schabesberger on 12.02.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * InfoItemCollector.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class InfoItemCollector {
+    private List<InfoItem> itemList = new Vector<>();
+    private List<Throwable> errors = new Vector<>();
+    private int serviceId = -1;
+
+    public InfoItemCollector(int serviceId) {
+        this.serviceId = serviceId;
+    }
+
+    public List<InfoItem> getItemList() {
+        return itemList;
+    }
+    public List<Throwable> getErrors() {
+        return errors;
+    }
+    protected void addFromCollector(InfoItemCollector otherC) throws ExtractionException {
+        if(serviceId != otherC.serviceId) {
+            throw new ExtractionException("Service Id does not equal: "
+                    + NewPipe.getNameOfService(serviceId)
+                    + " and " + NewPipe.getNameOfService(otherC.serviceId));
+        }
+        errors.addAll(otherC.errors);
+        itemList.addAll(otherC.itemList);
+    }
+    protected void addError(Exception e) {
+        errors.add(e);
+    }
+    protected void addItem(InfoItem item) {
+        itemList.add(item);
+    }
+    protected int getServiceId() {
+        return serviceId;
+    }
+}

MediaFormat.java

+package org.schabi.newpipe.extractor;
+
+/**
+ * Created by Adam Howard on 08/11/15.
+ *
+ * Copyright (c) Christian Schabesberger <chris.schabesberger@mailbox.org>
+ *     and Adam Howard <achdisposable1@gmail.com> 2015
+ *
+ * MediaFormat.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**Static data about various media formats support by Newpipe, eg mime type, extension*/
+
+public enum MediaFormat {
+    //video and audio combined formats
+    //           id      name    suffix  mime type
+    MPEG_4      (0x0,   "MPEG-4", "mp4", "video/mp4"),
+    v3GPP       (0x1,   "3GPP",   "3gp", "video/3gpp"),
+    WEBM        (0x2,   "WebM",  "webm", "video/webm"),
+    // audio formats
+    M4A         (0x3,   "m4a",   "m4a",  "audio/mp4"),
+    WEBMA       (0x4,   "WebM",  "webm", "audio/webm");
+
+    public final int id;
+    @SuppressWarnings("WeakerAccess")
+    public final String name;
+    @SuppressWarnings("WeakerAccess")
+    public final String suffix;
+    public final String mimeType;
+
+    MediaFormat(int id, String name, String suffix, String mimeType) {
+        this.id = id;
+        this.name = name;
+        this.suffix = suffix;
+        this.mimeType = mimeType;
+    }
+
+    /**Return the friendly name of the media format with the supplied id
+     * @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number.
+     * @return the friendly name of the MediaFormat associated with this ids,
+     * or an empty String if none match it.*/
+    public static String getNameById(int ident) {
+        for (MediaFormat vf : MediaFormat.values()) {
+            if(vf.id == ident) return vf.name;
+        }
+        return "";
+    }
+
+    /**Return the file extension of the media format with the supplied id
+     * @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number.
+     * @return the file extension of the MediaFormat associated with this ids,
+     * or an empty String if none match it.*/
+    public static String getSuffixById(int ident) {
+        for (MediaFormat vf : MediaFormat.values()) {
+            if(vf.id == ident) return vf.suffix;
+        }
+        return "";
+    }
+
+    /**Return the MIME type of the media format with the supplied id
+     * @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number.
+     * @return the MIME type of the MediaFormat associated with this ids,
+     * or an empty String if none match it.*/
+    public static String getMimeById(int ident) {
+        for (MediaFormat vf : MediaFormat.values()) {
+            if(vf.id == ident) return vf.mimeType;
+        }
+        return "";
+    }
+}

NewPipe.java

+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
+
+/**
+ * Created by Christian Schabesberger on 23.08.15.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * NewPipe.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**Provides access to the video streaming services supported by NewPipe.
+ * Currently only Youtube until the API becomes more stable.*/
+
+@SuppressWarnings("ALL")
+public class NewPipe {
+
+    private NewPipe() {
+    }
+
+    private static final String TAG = NewPipe.class.toString();
+
+
+    private static Downloader downloader = null;
+
+    public static StreamingService[] getServices() {
+        return ServiceList.serviceList;
+    }
+    public static StreamingService getService(int serviceId)throws ExtractionException {
+        for(StreamingService s : ServiceList.serviceList) {
+            if(s.getServiceId() == serviceId) {
+                return s;
+            }
+        }
+        return null;
+    }
+    public static StreamingService getService(String serviceName) throws ExtractionException {
+        return ServiceList.serviceList[getIdOfService(serviceName)];
+    }
+    public static String getNameOfService(int id) {
+        try {
+            return getService(id).getServiceInfo().name;
+        } catch (Exception e) {
+            System.err.println("Service id not known");
+            e.printStackTrace();
+            return "";
+        }
+    }
+    public static int getIdOfService(String serviceName) {
+        for(int i = 0; i < ServiceList.serviceList.length; i++) {
+            if(ServiceList.serviceList[i].getServiceInfo().name.equals(serviceName)) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    public static void init(Downloader d) {
+        downloader = d;
+    }
+
+    public static Downloader getDownloader() {
+        return downloader;
+    }
+
+    public static StreamingService getServiceByUrl(String url) {
+        for(StreamingService s : ServiceList.serviceList) {
+            if(s.getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) {
+                return s;
+            }
+        }
+        return null;
+    }
+}

Parser.java

+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Created by Christian Schabesberger on 02.02.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * Parser.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/** avoid using regex !!! */
+public class Parser {
+
+    private Parser() {
+    }
+
+    public static class RegexException extends ParsingException {
+        public RegexException(String message) {
+            super(message);
+        }
+    }
+
+    public static String matchGroup1(String pattern, String input) throws RegexException {
+        return matchGroup(pattern, input, 1);
+    }
+
+    public static String matchGroup(String pattern, String input, int group) throws RegexException {
+        Pattern pat = Pattern.compile(pattern);
+        Matcher mat = pat.matcher(input);
+        boolean foundMatch = mat.find();
+        if (foundMatch) {
+            return mat.group(group);
+        }
+        else {
+            //Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\"");
+            if(input.length() > 1024) {
+                throw new RegexException("failed to find pattern \""+pattern);
+            } else {
+                throw new RegexException("failed to find pattern \"" + pattern + " inside of " + input + "\"");
+            }
+        }
+    }
+
+    public static Map<String, String> compatParseMap(final String input) throws UnsupportedEncodingException {
+        Map<String, String> map = new HashMap<>();
+        for(String arg : input.split("&")) {
+            String[] splitArg = arg.split("=");
+            if(splitArg.length > 1) {
+                map.put(splitArg[0], URLDecoder.decode(splitArg[1], "UTF-8"));
+            } else {
+                map.put(splitArg[0], "");
+            }
+        }
+        return map;
+    }
+}

README.md

+NewPipe Extractor
+=================
+
+This is the system independent core of [NewPipe](https://github.com/TeamNewPipe/NewPipe).
+It can be used to create your own java based NewPipe client.

ServiceList.java

+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
+
+/**
+ * Created by the-scrabi on 18.02.17.
+ */
+
+class ServiceList {
+    public static final StreamingService[] serviceList = {
+            new YoutubeService(0)
+    };
+}

StreamingService.java

+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.search.SearchEngine;
+import org.schabi.newpipe.extractor.stream_info.StreamExtractor;
+
+import java.io.IOException;
+
+/**
+ * Created by Christian Schabesberger on 23.08.15.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * StreamingService.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public abstract class StreamingService {
+    public class ServiceInfo {
+        public String name = "";
+    }
+
+    public enum LinkType {
+        NONE,
+        STREAM,
+        CHANNEL,
+        PLAYLIST
+    }
+
+    private int serviceId;
+
+    public StreamingService(int id) {
+        serviceId = id;
+    }
+
+    public abstract ServiceInfo getServiceInfo();
+
+    public abstract StreamExtractor getExtractorInstance(String url)
+            throws IOException, ExtractionException;
+    public abstract SearchEngine getSearchEngineInstance();
+    public abstract UrlIdHandler getStreamUrlIdHandlerInstance();
+    public abstract UrlIdHandler getChannelUrlIdHandlerInstance();
+    public abstract ChannelExtractor getChannelExtractorInstance(String url, int page)
+            throws ExtractionException, IOException;
+    public abstract SuggestionExtractor getSuggestionExtractorInstance();
+
+    public final int getServiceId() {
+        return serviceId;
+    }
+
+    /**
+     * figure out where the link is pointing to (a channel, video, playlist, etc.)
+     */