diff --git a/Cargo.toml b/Cargo.toml index 9eb5326..47e665a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ authors = ["madiele92@gmail.com"] edition = "2021" name = "vod2pod-rss" -version = "1.1.2" +version = "1.2.0" [lib] path = "src/lib.rs" @@ -14,17 +14,17 @@ path = "src/main.rs" [dependencies] actix-rt = "=2.9.0" google-youtube3 = "=5.0.3" -actix-web = "=4.4.1" +actix-web = "=4.5.1" async-trait = "=0.1.77" url = { version="=2.5.0", features = ["serde"]} futures = "=0.3.30" log = "=0.4.20" -regex = "=1.10.2" -reqwest = { version = "=0.11.23", features = ["json"] } -serde = "=1.0.195" -serde_json = "=1.0.111" -tokio = { version = "=1.35.1", features = ["macros", "process"]} -uuid = { version= "=1.6.1", features = ["v4", "serde"]} +regex = "=1.10.3" +reqwest = { version = "=0.11.24", features = ["json"] } +serde = "=1.0.197" +serde_json = "=1.0.114" +tokio = { version = "=1.36.0", features = ["macros", "process"]} +uuid = { version= "=1.7.0", features = ["v4", "serde"]} genawaiter = {version = "=0.99", features = ["futures03"] } openssl = { version = "*", features = ["vendored"] } #this is here just to make cross-compiling work during github actions rss = { version = "=2.0", features = ["serde"] } @@ -32,9 +32,9 @@ eyre = "=0.6" simple_logger = "=4.3" redis = { version = "=0.24", features = ["tokio-comp"] } mime = "=0.3.17" -cached = { version = "=0.47.0", features = ["redis_tokio"] } +cached = { version = "=0.49.2", features = ["redis_tokio"] } iso8601-duration = "=0.2.0" -chrono = "=0.4.31" +chrono = "=0.4.34" feed-rs = "=1.4.0" [dev-dependencies] diff --git a/Dockerfile b/Dockerfile index da49571..cba0f0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # by using --platform=$BUILDPLATFORM we force the build step # to always run on the native architecture of the build machine # making the build time shorter -FROM --platform=$BUILDPLATFORM rust:1.74 as builder +FROM --platform=$BUILDPLATFORM rust:1.75 as builder ARG BUILDPLATFORM ARG TARGETPLATFORM diff --git a/README.md b/README.md index ed9009b..0e0d96b 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ run this inside the folder with `docker-compose.yml` `sudo docker compose pull && sudo docker compose up -d` -then run this to delete the old version form your system (note: this will also delete any other unused image you have) +then run this to delete the old version from your system (note: this will also delete any other unused image you have) `sudo docker system prune` diff --git a/docker-compose.yml b/docker-compose.yml index 70a0c2c..14131ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,10 @@ services: #change "latest" to "X.X.X" to pin a version es: "1.0.4" will force the image to use to version 1.0.4, if you do please watch the repo for updates (tutorial in README.md) #change "latest" to "beta" if you want to test yet unreleased fixes/features (expect bugs and broken builds from time to time) image: madiele/vod2pod-rss:latest + # uncomment to build vod2pod from scratch, only do this if your architecture is not supported + #build: + # dockerfile: ./Dockerfile + # context: https://github.com/madiele/vod2pod-rss.git depends_on: - redis restart: unless-stopped diff --git a/src/configs/mod.rs b/src/configs/mod.rs index 300b8eb..02aff54 100644 --- a/src/configs/mod.rs +++ b/src/configs/mod.rs @@ -15,6 +15,7 @@ pub enum ConfName { RedisUrl, Mp3Bitrate, YoutubeApiKey, + YouutbeMaxResults, TwitchClientId, TwitchSecretKey, TranscodingEnabled, @@ -102,6 +103,9 @@ impl Conf for EnvConf { ConfName::PeerTubeValidHosts => { Ok(std::env::var("PEERTUBE_VALID_DOMAINS").unwrap_or_else(|_| "".to_string())) } + ConfName::YouutbeMaxResults => { + Ok(std::env::var("YOUTUBE_MAX_RESULTS").unwrap_or_else(|_| "300".to_string())) + } } } } diff --git a/src/provider/twitch.rs b/src/provider/twitch.rs index 5ec2b8c..7962ce1 100644 --- a/src/provider/twitch.rs +++ b/src/provider/twitch.rs @@ -56,7 +56,7 @@ impl MediaProvider for TwitchProvider { .data; let channel = channels - .get(0) + .first() .ok_or_else(|| eyre::eyre!("No twitch user found"))?; debug!("fetched twitch channel: {:?}", channel); diff --git a/src/provider/youtube.rs b/src/provider/youtube.rs index 2140e28..b8bdc53 100644 --- a/src/provider/youtube.rs +++ b/src/provider/youtube.rs @@ -160,7 +160,9 @@ async fn fetch_from_api(id: IdType, api_key: String) -> eyre::Result<(Channel, V let rss_channel = build_channel_from_playlist(playlist); - let items = fetch_playlist_items(&playlist_id, &api_key).await?; + let max_fetched_items: usize = + conf().get(ConfName::YouutbeMaxResults).unwrap().parse()?; + let items = fetch_playlist_items(&playlist_id, &api_key, max_fetched_items).await?; let duration_map = create_duration_url_map(&items, &api_key).await?; @@ -183,7 +185,9 @@ async fn fetch_from_api(id: IdType, api_key: String) -> eyre::Result<(Channel, V let rss_channel = build_channel_from_yt_channel(channel); - let items = fetch_playlist_items(&upload_playlist, &api_key).await?; + let max_fetched_items: usize = + conf().get(ConfName::YouutbeMaxResults).unwrap().parse()?; + let items = fetch_playlist_items(&upload_playlist, &api_key, max_fetched_items).await?; let duration_map = create_duration_url_map(&items, &api_key).await?; @@ -355,37 +359,51 @@ fn build_channel_items_from_playlist( async fn fetch_playlist_items( playlist_id: &String, api_key: &String, + max_fetched_items: usize, ) -> eyre::Result> { let hub = get_youtube_hub(); - let max_consecutive_requests = 5; - let mut fetched_playlist_items: Vec = - Vec::with_capacity(max_consecutive_requests * 50); + let max_consecutive_requests = (max_fetched_items / 50) + 1; + let mut fetched_playlist_items: Vec = Vec::with_capacity(max_fetched_items); let mut request_count = 0; let mut next_page_token: Option = None; debug!("fetching items from playlist {}", playlist_id); loop { + let remaining_items = max_fetched_items - fetched_playlist_items.len(); + let items_to_fetch = if remaining_items > 50 { + 50 + } else { + remaining_items + }; + let mut playlist_items_request = hub .playlist_items() .list(&vec!["snippet".into()]) .playlist_id(playlist_id) .param("key", api_key) - .max_results(50); + .max_results(items_to_fetch.try_into()?); + if let Some(ref next_page_token) = next_page_token { playlist_items_request = playlist_items_request.page_token(next_page_token.as_str()); } + let response = playlist_items_request.doit().await?; - fetched_playlist_items.extend(response.1.items.ok_or(eyre!("playlist has no items"))?); + fetched_playlist_items.extend( + response + .1 + .items + .ok_or(eyre!("playlist object has no items field"))?, + ); next_page_token = response.1.next_page_token; - request_count += 1; - if next_page_token.is_none() || request_count > max_consecutive_requests { + if next_page_token.is_none() || request_count == max_consecutive_requests { info!( - "fetched {} items, channel too large, stopping", + "fetched {} items, max items reached or no more items to fetch", fetched_playlist_items.len() ); break; } + request_count += 1; } info!( "fetched {} items, in {} requests", @@ -671,13 +689,13 @@ mod tests { #[tokio::test] async fn test_build_items_for_playlist_requires_api_key() { - let id = "PLJmimp-uZX42T7ONp1FLXQDJrRxZ-_1Ct".to_string(); + let id = "UUXuqSBlHAE6Xw-yeJA0Tunw".to_string(); let api_key = conf().get(ConfName::YoutubeApiKey).unwrap(); let playlist = fetch_playlist(id, &api_key).await.unwrap(); println!("{:?}", &playlist.clone().id.unwrap().clone()); - let items = fetch_playlist_items(&playlist.id.unwrap(), &api_key) + let items = fetch_playlist_items(&playlist.id.unwrap(), &api_key, 300) .await .unwrap(); @@ -685,6 +703,57 @@ mod tests { assert!(!items.is_empty()) } + #[tokio::test] + async fn test_less_than_50_items_requires_api_key() { + let id = "UUXuqSBlHAE6Xw-yeJA0Tunw".to_string(); + let api_key = conf().get(ConfName::YoutubeApiKey).unwrap(); + + let playlist = fetch_playlist(id, &api_key).await.unwrap(); + + println!("{:?}", &playlist.clone().id.unwrap().clone()); + let items = fetch_playlist_items(&playlist.id.unwrap(), &api_key, 13) + .await + .unwrap(); + + println!("{:?}", items); + assert!(!items.is_empty()); + assert_eq!(items.len(), 13) + } + + #[tokio::test] + async fn test_less_than_300_items_requires_api_key() { + let id = "UUXuqSBlHAE6Xw-yeJA0Tunw".to_string(); + let api_key = conf().get(ConfName::YoutubeApiKey).unwrap(); + + let playlist = fetch_playlist(id, &api_key).await.unwrap(); + + println!("{:?}", &playlist.clone().id.unwrap().clone()); + let items = fetch_playlist_items(&playlist.id.unwrap(), &api_key, 50) + .await + .unwrap(); + + println!("{:?}", items); + assert!(!items.is_empty()); + assert_eq!(items.len(), 50) + } + + #[tokio::test] + async fn test_more_than_300_items_requires_api_key() { + let id = "UUXuqSBlHAE6Xw-yeJA0Tunw".to_string(); + let api_key = conf().get(ConfName::YoutubeApiKey).unwrap(); + + let playlist = fetch_playlist(id, &api_key).await.unwrap(); + + println!("{:?}", &playlist.clone().id.unwrap().clone()); + let items = fetch_playlist_items(&playlist.id.unwrap(), &api_key, 600) + .await + .unwrap(); + + println!("{:?}", items); + assert!(!items.is_empty()); + assert_eq!(items.len(), 600) + } + #[test(tokio::test)] async fn test_build_channel_for_playlist_requires_api_key() { let id = "PLJmimp-uZX42T7ONp1FLXQDJrRxZ-_1Ct".to_string(); diff --git a/src/rss_transcodizer/mod.rs b/src/rss_transcodizer/mod.rs index 8400cf4..9060f7b 100644 --- a/src/rss_transcodizer/mod.rs +++ b/src/rss_transcodizer/mod.rs @@ -5,6 +5,7 @@ use std::time::Duration; use eyre::eyre; use log::debug; use reqwest::Url; +use rss::extension::itunes::ITunesCategory; use rss::Channel; use rss::{Enclosure, Item}; @@ -16,6 +17,11 @@ pub fn inject_vod2pod_customizations( ) -> eyre::Result { let mut injected_feed = Channel::read_from(rss_body.as_bytes())?; injected_feed.set_generator(Some("generated by vod2pod-rss".to_string())); + if let Some(ref mut itunes) = injected_feed.itunes_ext { + let mut default_category = ITunesCategory::default(); + default_category.set_text("Technology"); + itunes.set_categories(vec![default_category]); + } let mut namespaces = BTreeMap::new(); namespaces.insert( "rss".to_string(), @@ -25,7 +31,12 @@ pub fn inject_vod2pod_customizations( "itunes".to_string(), "http://www.itunes.com/dtds/podcast-1.0.dtd".to_string(), ); + namespaces.insert( + "content".to_string(), + "http://purl.org/rss/1.0/modules/content/".to_string(), + ); injected_feed.set_namespaces(namespaces); + injected_feed.set_language("en-US".to_string()); injected_feed .items_mut() .iter_mut() diff --git a/src/server/mod.rs b/src/server/mod.rs index 3cd0229..1d0cec8 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,6 +1,8 @@ use std::{collections::HashMap, net::TcpListener, time::Instant}; -use actix_web::{dev::Server, guard, middleware, web, App, HttpRequest, HttpResponse, HttpServer}; +use actix_web::{ + dev::Server, guard, http, middleware, web, App, HttpRequest, HttpResponse, HttpServer, +}; use log::{debug, error, info, warn}; use regex::Regex; use serde::Deserialize; @@ -23,12 +25,21 @@ pub fn spawn_server(listener: TcpListener) -> eyre::Result { .service( web::scope(&root) .service( - web::resource("transcode_media/to_mp3") + web::resource("transcode_media/to.mp3") .name("transcode_mp3") - .guard(guard::Get()) + .guard(guard::Any(guard::Get()).or(guard::Head())) + .to(transcode_to_mp3), + ) + .service( + //this is an old URL used in old vod2pod versions that did not work with + //itunes kept for backwards compatiility + web::resource("transcode_media/to_mp3") + .name("transcode_mp3_obsolete") + .guard(guard::Any(guard::Get()).or(guard::Head())) .to(transcode_to_mp3), ) .route("transcodize_rss", web::get().to(transcodize_rss)) + .route("transcodize_rss", web::head().to(transcodize_rss)) .route("health", web::get().to(health)) .route("/", web::get().to(index)) .route("", web::get().to(index)), @@ -63,6 +74,10 @@ async fn transcodize_rss( req: HttpRequest, query: web::Query>, ) -> HttpResponse { + if req.method() == http::Method::HEAD { + return HttpResponse::Ok().finish(); + } + let start_time = Instant::now(); let should_transcode = match conf().get(ConfName::TranscodingEnabled) { @@ -258,6 +273,17 @@ async fn transcode_to_mp3(req: HttpRequest, query: web::Query) }; debug!("seconds: {duration_secs}, bitrate: {bitrate}"); + if req.method() == http::Method::HEAD { + return HttpResponse::Ok() + .insert_header(("Accept-Ranges", "bytes")) + .insert_header(( + "Content-Range", + format!("bytes {start_bytes}-{end_bytes}/{total_streamable_bytes}"), + )) + .content_type(codec.get_mime_type_str()) + .finish(); + } + match Transcoder::new(&ffmpeg_paramenters).await { Ok(transcoder) => { let stream = transcoder.get_transcode_stream();