Unverified Commit b14b4ff5 authored by spikecodes's avatar spikecodes
Browse files

Rewrite server and client using Hyper

parent 4b1195f2
......@@ -3,19 +3,23 @@ name = "libreddit"
description = " Alternative private front-end to Reddit"
license = "AGPL-3.0"
repository = "https://github.com/spikecodes/libreddit"
version = "0.4.2"
version = "0.5.0"
authors = ["spikecodes <19519553+spikecodes@users.noreply.github.com>"]
edition = "2018"
[dependencies]
askama = { version = "0.10.5", default-features = false }
async-recursion = "0.3.2"
async-std = { version = "1.9.0", features = ["attributes"] }
async-tls = { version = "0.11.0", default-features = false, features = ["client"] }
cached = "0.23.0"
clap = { version = "2.33.3", default-features = false }
regex = "1.4.4"
regex = "1.4.5"
serde = { version = "1.0.124", features = ["derive"] }
cookie = "0.15.0"
futures-lite = "1.11.3"
hyper = { version = "0.14.4", features = ["full"] }
hyper-rustls = "0.22.1"
route-recognizer = "0.3.0"
serde_json = "1.0.64"
tide = { version = "0.16.0", default-features = false, features = ["h1-server", "cookies"] }
time = "0.2.25"
tokio = { version = "1.3.0", features = ["full"] }
time = "0.2.26"
url = "2.2.1"
use cached::proc_macro::cached;
use futures_lite::{future::Boxed, FutureExt};
use hyper::{body::Buf, client, Body, Request, Response, Uri};
use serde_json::Value;
use std::{result::Result, str::FromStr};
// use async_recursion::async_recursion;
use crate::server::RequestExt;
pub async fn proxy(req: Request<Body>, format: &str) -> Result<Response<Body>, String> {
let mut url = format.to_string();
for (name, value) in req.params().iter() {
url = url.replace(&format!("{{{}}}", name), value);
}
stream(&url).await
}
async fn stream(url: &str) -> Result<Response<Body>, String> {
// First parameter is target URL (mandatory).
let url = Uri::from_str(url).unwrap();
// Prepare the HTTPS connector.
let https = hyper_rustls::HttpsConnector::with_native_roots();
// Build the hyper client from the HTTPS connector.
let client: client::Client<_, hyper::Body> = client::Client::builder().build(https);
client
.get(url)
.await
.map(|mut res| {
let mut rm = |key: &str| res.headers_mut().remove(key);
rm("access-control-expose-headers");
rm("server");
rm("vary");
rm("etag");
rm("x-cdn");
rm("x-cdn-client-region");
rm("x-cdn-name");
rm("x-cdn-server-region");
res
})
.map_err(|e| e.to_string())
}
fn request(url: String) -> Boxed<Result<Response<Body>, String>> {
// Prepare the HTTPS connector.
let https = hyper_rustls::HttpsConnector::with_native_roots();
// Build the hyper client from the HTTPS connector.
let client: client::Client<_, hyper::Body> = client::Client::builder().build(https);
let req = |uri: String| {
Request::builder()
.method("GET")
.uri(&uri)
.header("User-Agent", format!("web:libreddit:{}", env!("CARGO_PKG_VERSION")))
.header("Host", "www.reddit.com")
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
.header("Accept-Language", "en-US,en;q=0.5")
.header("Connection", "keep-alive")
.body(Body::empty())
.expect("request builder")
};
async move {
match client.request(req(url)).await {
Ok(response) => {
if response.status().to_string().starts_with('3') {
request(response.headers().get("Location").unwrap().to_str().unwrap_or_default().to_string()).await
} else {
Ok(response)
}
}
Err(e) => Err(e.to_string()),
}
}
.boxed()
}
// Make a request to a Reddit API and parse the JSON response
#[cached(size = 100, time = 30, result = true)]
pub async fn json(path: String) -> Result<Value, String> {
// Build Reddit url from path
let url = format!("https://www.reddit.com{}", path);
// Closure to quickly build errors
let err = |msg: &str, e: String| -> Result<Value, String> {
eprintln!("{} - {}: {}", url, msg, e);
Err(msg.to_string())
};
// Fetch the url...
match request(url.clone()).await {
Ok(response) => {
// asynchronously aggregate the chunks of the body
match hyper::body::aggregate(response).await {
Ok(body) => {
// Parse the response from Reddit as JSON
match serde_json::from_reader(body.reader()) {
Ok(value) => {
let json: Value = value;
// If Reddit returned an error
if json["error"].is_i64() {
Err(
json["reason"]
.as_str()
.unwrap_or_else(|| {
json["message"].as_str().unwrap_or_else(|| {
eprintln!("{} - Error parsing reddit error", url);
"Error parsing reddit error"
})
})
.to_string(),
)
} else {
Ok(json)
}
}
Err(e) => err("Failed to parse page JSON data", e.to_string()),
}
}
Err(e) => err("Failed receiving JSON body from Reddit", e.to_string()),
}
}
Err(e) => err("Couldn't send request to Reddit", e.to_string()),
}
}
......@@ -11,7 +11,6 @@
// Reference local files
mod post;
mod proxy;
mod search;
mod settings;
mod subreddit;
......@@ -19,99 +18,70 @@ mod user;
mod utils;
// Import Crates
use clap::{App, Arg};
use proxy::handler;
use tide::{
utils::{async_trait, After},
Middleware, Next, Request, Response,
};
use utils::{error, redirect};
use clap::{App as cli, Arg};
// Build middleware
struct HttpsRedirect<HttpsOnly>(HttpsOnly);
struct NormalizePath;
#[async_trait]
impl<State, HttpsOnly> Middleware<State> for HttpsRedirect<HttpsOnly>
where
State: Clone + Send + Sync + 'static,
HttpsOnly: Into<bool> + Copy + Send + Sync + 'static,
{
async fn handle(&self, request: Request<State>, next: Next<'_, State>) -> tide::Result {
let secure = request.url().scheme() == "https";
if self.0.into() && !secure {
let mut secured = request.url().to_owned();
secured.set_scheme("https").unwrap_or_default();
Ok(redirect(secured.to_string()))
} else {
Ok(next.run(request).await)
}
}
}
use futures_lite::FutureExt;
use hyper::{Body, Request, Response};
#[async_trait]
impl<State: Clone + Send + Sync + 'static> Middleware<State> for NormalizePath {
async fn handle(&self, request: Request<State>, next: Next<'_, State>) -> tide::Result {
let path = request.url().path();
let query = request.url().query().unwrap_or_default();
if path.ends_with('/') {
Ok(next.run(request).await)
} else {
let normalized = if query.is_empty() {
format!("{}/", path.replace("//", "/"))
} else {
format!("{}/?{}", path.replace("//", "/"), query)
};
Ok(redirect(normalized))
}
}
}
mod client;
use client::proxy;
use server::RequestExt;
use utils::{error, redirect};
mod server;
// Create Services
// Required for the manifest to be valid
async fn pwa_logo(_req: Request<()>) -> tide::Result {
Ok(Response::builder(200).content_type("image/png").body(include_bytes!("../static/logo.png").as_ref()).build())
async fn pwa_logo() -> Result<Response<Body>, String> {
Ok(
Response::builder()
.status(200)
.header("content-type", "image/png")
.body(include_bytes!("../static/logo.png").as_ref().into())
.unwrap_or_default(),
)
}
// Required for iOS App Icons
async fn iphone_logo(_req: Request<()>) -> tide::Result {
async fn iphone_logo() -> Result<Response<Body>, String> {
Ok(
Response::builder(200)
.content_type("image/png")
.body(include_bytes!("../static/apple-touch-icon.png").as_ref())
.build(),
Response::builder()
.status(200)
.header("content-type", "image/png")
.body(include_bytes!("../static/apple-touch-icon.png").as_ref().into())
.unwrap_or_default(),
)
}
async fn favicon(_req: Request<()>) -> tide::Result {
async fn favicon() -> Result<Response<Body>, String> {
Ok(
Response::builder(200)
.content_type("image/vnd.microsoft.icon")
Response::builder()
.status(200)
.header("content-type", "image/vnd.microsoft.icon")
.header("Cache-Control", "public, max-age=1209600, s-maxage=86400")
.body(include_bytes!("../static/favicon.ico").as_ref())
.build(),
.body(include_bytes!("../static/favicon.ico").as_ref().into())
.unwrap_or_default(),
)
}
async fn resource(body: &str, content_type: &str, cache: bool) -> tide::Result {
let mut res = Response::new(200);
async fn resource(body: &str, content_type: &str, cache: bool) -> Result<Response<Body>, String> {
let mut res = Response::builder()
.status(200)
.header("content-type", content_type)
.body(body.to_string().into())
.unwrap_or_default();
if cache {
res.insert_header("Cache-Control", "public, max-age=1209600, s-maxage=86400");
res.headers_mut().insert("Cache-Control", "public, max-age=1209600, s-maxage=86400".parse().unwrap());
}
res.set_content_type(content_type);
res.set_body(body);
Ok(res)
}
#[async_std::main]
async fn main() -> tide::Result<()> {
let matches = App::new("Libreddit")
#[tokio::main]
async fn main() {
let matches = cli::new("Libreddit")
.version(env!("CARGO_PKG_VERSION"))
.about("Private front-end for Reddit written in Rust ")
.arg(
......@@ -136,142 +106,126 @@ async fn main() -> tide::Result<()> {
Arg::with_name("redirect-https")
.short("r")
.long("redirect-https")
.help("Redirect all HTTP requests to HTTPS")
.help("Redirect all HTTP requests to HTTPS (no longer functional)")
.takes_value(false),
)
.get_matches();
let address = matches.value_of("address").unwrap_or("0.0.0.0");
let port = matches.value_of("port").unwrap_or("8080");
let force_https = matches.is_present("redirect-https");
let _force_https = matches.is_present("redirect-https");
let listener = format!("{}:{}", address, port);
println!("Starting Libreddit...");
// Start HTTP server
let mut app = tide::new();
// Begin constructing a server
let mut app = server::Server::new();
// Redirect to HTTPS if "--redirect-https" enabled
app.with(HttpsRedirect(force_https));
// Append trailing slash and remove double slashes
app.with(NormalizePath);
// Apply default headers for security
app.with(After(|mut res: Response| async move {
res.insert_header("Referrer-Policy", "no-referrer");
res.insert_header("X-Content-Type-Options", "nosniff");
res.insert_header("X-Frame-Options", "DENY");
res.insert_header(
"Content-Security-Policy",
"default-src 'none'; manifest-src 'self'; media-src 'self'; style-src 'self' 'unsafe-inline'; base-uri 'none'; img-src 'self' data:; form-action 'self'; frame-ancestors 'none';",
);
Ok(res)
}));
// Define default headers (added to all responses)
app.default_headers = headers! {
"Referrer-Policy" => "no-referrer",
"X-Content-Type-Options" => "nosniff",
"X-Frame-Options" => "DENY",
"Content-Security-Policy" => "default-src 'none'; manifest-src 'self'; media-src 'self'; style-src 'self' 'unsafe-inline'; base-uri 'none'; img-src 'self' data:; form-action 'self'; frame-ancestors 'none';"
};
// Read static files
app.at("/style.css/").get(|_| resource(include_str!("../static/style.css"), "text/css", false));
app.at("/style.css").get(|_| resource(include_str!("../static/style.css"), "text/css", false).boxed());
app
.at("/manifest.json/")
.get(|_| resource(include_str!("../static/manifest.json"), "application/json", false));
app.at("/robots.txt/").get(|_| resource("User-agent: *\nAllow: /", "text/plain", true));
app.at("/favicon.ico/").get(favicon);
app.at("/logo.png/").get(pwa_logo);
app.at("/touch-icon-iphone.png/").get(iphone_logo);
app.at("/apple-touch-icon.png/").get(iphone_logo);
.at("/manifest.json")
.get(|_| resource(include_str!("../static/manifest.json"), "application/json", false).boxed());
app.at("/robots.txt").get(|_| resource("User-agent: *\nAllow: /", "text/plain", true).boxed());
app.at("/favicon.ico").get(|_| favicon().boxed());
app.at("/logo.png").get(|_| pwa_logo().boxed());
app.at("/touch-icon-iphone.png").get(|_| iphone_logo().boxed());
app.at("/apple-touch-icon.png").get(|_| iphone_logo().boxed());
// Proxy media through Libreddit
app
.at("/vid/:id/:size/") /* */
.get(|req| handler(req, "https://v.redd.it/{}/DASH_{}", vec!["id", "size"]));
app
.at("/img/:id/") /* */
.get(|req| handler(req, "https://i.redd.it/{}", vec!["id"]));
app
.at("/thumb/:point/:id/") /* */
.get(|req| handler(req, "https://{}.thumbs.redditmedia.com/{}", vec!["point", "id"]));
app
.at("/emoji/:id/:name/") /* */
.get(|req| handler(req, "https://emoji.redditmedia.com/{}/{}", vec!["id", "name"]));
app
.at("/preview/:loc/:id/:query/")
.get(|req| handler(req, "https://{}view.redd.it/{}?{}", vec!["loc", "id", "query"]));
app
.at("/style/*path/") /* */
.get(|req| handler(req, "https://styles.redditmedia.com/{}", vec!["path"]));
app
.at("/static/*path/") /* */
.get(|req| handler(req, "https://www.redditstatic.com/{}", vec!["path"]));
app.at("/vid/:id/:size").get(|r| proxy(r, "https://v.redd.it/{id}/DASH_{size}").boxed());
app.at("/img/:id").get(|r| proxy(r, "https://i.redd.it/{id}").boxed());
app.at("/thumb/:point/:id").get(|r| proxy(r, "https://{point}.thumbs.redditmedia.com/{id}").boxed());
app.at("/emoji/:id/:name").get(|r| proxy(r, "https://emoji.redditmedia.com/{id}/{name}").boxed());
app.at("/preview/:loc/:id/:query").get(|r| proxy(r, "https://{loc}view.redd.it/{id}?{query}").boxed());
app.at("/style/*path").get(|r| proxy(r, "https://styles.redditmedia.com/{path}").boxed());
app.at("/static/*path").get(|r| proxy(r, "https://www.redditstatic.com/{path}").boxed());
// Browse user profile
app.at("/u/:name/").get(user::profile);
app.at("/u/:name/comments/:id/:title/").get(post::item);
app.at("/u/:name/comments/:id/:title/:comment_id/").get(post::item);
app
.at("/u/:name")
.get(|r| async move { Ok(redirect(format!("/user/{}", r.param("name").unwrap_or_default()))) }.boxed());
app.at("/u/:name/comments/:id/:title").get(|r| post::item(r).boxed());
app.at("/u/:name/comments/:id/:title/:comment_id").get(|r| post::item(r).boxed());
app.at("/user/:name/").get(user::profile);
app.at("/user/:name/comments/:id/").get(post::item);
app.at("/user/:name/comments/:id/:title/").get(post::item);
app.at("/user/:name/comments/:id/:title/:comment_id/").get(post::item);
app.at("/user/:name").get(|r| user::profile(r).boxed());
app.at("/user/:name/comments/:id").get(|r| post::item(r).boxed());
app.at("/user/:name/comments/:id/:title").get(|r| post::item(r).boxed());
app.at("/user/:name/comments/:id/:title/:comment_id").get(|r| post::item(r).boxed());
// Configure settings
app.at("/settings/").get(settings::get).post(settings::set);
app.at("/settings/restore/").get(settings::restore);
app.at("/settings").get(|r| settings::get(r).boxed()).post(|r| settings::set(r).boxed());
app.at("/settings/restore").get(|r| settings::restore(r).boxed());
// Subreddit services
app.at("/r/:sub/").get(subreddit::community);
app.at("/r/:sub").get(|r| subreddit::community(r).boxed());
app.at("/r/:sub/subscribe/").post(subreddit::subscriptions);
app.at("/r/:sub/unsubscribe/").post(subreddit::subscriptions);
app.at("/r/:sub/subscribe").post(|r| subreddit::subscriptions(r).boxed());
app.at("/r/:sub/unsubscribe").post(|r| subreddit::subscriptions(r).boxed());
app.at("/r/:sub/comments/:id/").get(post::item);
app.at("/r/:sub/comments/:id/:title/").get(post::item);
app.at("/r/:sub/comments/:id/:title/:comment_id/").get(post::item);
app.at("/r/:sub/comments/:id").get(|r| post::item(r).boxed());
app.at("/r/:sub/comments/:id/:title").get(|r| post::item(r).boxed());
app.at("/r/:sub/comments/:id/:title/:comment_id").get(|r| post::item(r).boxed());
app.at("/r/:sub/search/").get(search::find);
app.at("/r/:sub/search").get(|r| search::find(r).boxed());
app.at("/r/:sub/wiki/").get(subreddit::wiki);
app.at("/r/:sub/wiki/:page/").get(subreddit::wiki);
app.at("/r/:sub/w/").get(subreddit::wiki);
app.at("/r/:sub/w/:page/").get(subreddit::wiki);
app.at("/r/:sub/wiki/").get(|r| subreddit::wiki(r).boxed());
app.at("/r/:sub/wiki/:page").get(|r| subreddit::wiki(r).boxed());
app.at("/r/:sub/w").get(|r| subreddit::wiki(r).boxed());
app.at("/r/:sub/w/:page").get(|r| subreddit::wiki(r).boxed());
app.at("/r/:sub/:sort/").get(subreddit::community);
app.at("/r/:sub/:sort").get(|r| subreddit::community(r).boxed());
// Comments handler
app.at("/comments/:id/").get(post::item);
app.at("/comments/:id/").get(|r| post::item(r).boxed());
// Front page
app.at("/").get(subreddit::community);
app.at("/").get(|r| subreddit::community(r).boxed());
// View Reddit wiki
app.at("/w/").get(subreddit::wiki);
app.at("/w/:page/").get(subreddit::wiki);
app.at("/wiki/").get(subreddit::wiki);
app.at("/wiki/:page/").get(subreddit::wiki);
app.at("/w").get(|r| subreddit::wiki(r).boxed());
app.at("/w/:page").get(|r| subreddit::wiki(r).boxed());
app.at("/wiki").get(|r| subreddit::wiki(r).boxed());
app.at("/wiki/:page").get(|r| subreddit::wiki(r).boxed());
// Search all of Reddit
app.at("/search/").get(search::find);
app.at("/search").get(|r| search::find(r).boxed());
// Handle about pages
app.at("/about/").get(|req| error(req, "About pages aren't here yet".to_string()));
app.at("/:id/").get(|req: Request<()>| async {
match req.param("id") {
// Sort front page
Ok("best") | Ok("hot") | Ok("new") | Ok("top") | Ok("rising") | Ok("controversial") => subreddit::community(req).await,
// Short link for post
Ok(id) if id.len() > 4 && id.len() < 7 => post::item(req).await,
// Error message for unknown pages
_ => error(req, "Nothing here".to_string()).await,
app.at("/about").get(|req| error(req, "About pages aren't added yet".to_string()).boxed());
app.at("/:id").get(|req: Request<Body>| {
async {
match req.param("id") {
// Sort front page
// Some("best") | Some("hot") | Some("new") | Some("top") | Some("rising") | Some("controversial") => subreddit::community(req).await,
// Short link for post
Some(id) if id.len() > 4 && id.len() < 7 => post::item(req).await,
// Error message for unknown pages
_ => error(req, "Nothing here".to_string()).await,
}
}
.boxed()
});
// Default service in case no routes match
app.at("*").get(|req| error(req, "Nothing here".to_string()));
app.at("/*").get(|req| error(req, "Nothing here".to_string()).boxed());
println!("Running Libreddit v{} on {}!", env!("CARGO_PKG_VERSION"), listener);
app.listen(&listener).await?;
let server = app.listen(listener);
Ok(())
// Run this server for... forever!
if let Err(e) = server.await {
eprintln!("Server error: {}", e);
}
}
// CRATES
use crate::client::json;
use crate::esc;
use crate::utils::{
cookie, error, format_num, format_url, param, request, rewrite_urls, template, time, val, Author, Comment, Flags, Flair, FlairPart, Media, Post, Preferences,
};
use tide::Request;
use crate::server::RequestExt;
use crate::utils::{cookie, error, format_num, format_url, param, rewrite_urls, template, time, val, Author, Comment, Flags, Flair, FlairPart, Media, Post, Preferences};
use hyper::{Body, Request, Response};
use async_recursion::async_recursion;
......@@ -20,9 +20,9 @@ struct PostTemplate {
single_thread: bool,
}
pub async fn item(req: Request<()>) -> tide::Result {
pub async fn item(req: Request<Body>) -> Result<Response<Body>, String> {
// Build Reddit API path
let mut path: String = format!("{}.json?{}&raw_json=1", req.url().path(), req.url().query().unwrap_or_default());
let mut path: String = format!("{}.json?{}&raw_json=1", req.uri().path(), req.uri().query().unwrap_or_default());
// Set sort to sort query parameter
let mut sort: String = param(&path, "sort");
......@@ -33,23 +33,23 @@ pub async fn item(req: Request<()>) -> tide::Result {
// If there's no sort query but there's a default sort, set sort to default_sort
if sort.is_empty() && !default_sort.is_empty() {
sort = default_sort;
path = format!("{}.json?{}&sort={}&raw_json=1", req.url().path(), req.url().query().unwrap_or_default(), sort);
path = format!("{}.json?{}&sort={}&raw_json=1", req.uri().path(), req.uri().query().unwrap_or_default(), sort);
}
// Log the post ID being fetched in debug mode
#[cfg(debug_assertions)]
dbg!(req.param("id").unwrap_or(""));
dbg!(req.param("id").unwrap_or_default());
let single_thread = &req.param("comment_id").is_ok();
let single_thread = &req.param("comment_id").is_some();
let highlighted_comment = &req.param("comment_id").unwrap_or_default();
// Send a request to the url, receive JSON in response
match request(path).await {
match json(path).await {
// Otherwise, grab the JSON output from the request
Ok(res) => {
// Parse the JSON into Post and Comment structs
let post = parse_post(&res[0]).await;
let comments = parse_comments(&res[1], &post.permalink, &post.author.name, *highlighted_comment).await;
let comments = parse_comments(&res[1], &post.permalink, &post.author.name, highlighted_comment).await;
// Use the Post and Comment structs to generate a website to show users
template(PostTemplate {
......
use async_std::{io, net::TcpStream, prelude::*};
use async_tls::TlsConnector;
use tide::{http::url::Url, Request, Response};
/// Handle tide routes to proxy by parsing `params` from `req`uest.
pub async fn handler(req: Request<()>, format: &str, params: Vec<&str>) -> tide::Result {
let mut url = format.to_string();
for name in params {
let param