Merge pull request #152 from madiele/main

version 1.2

Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["madiele92@gmail.com"]
 edition = "2021"
 name = "vod2pod-rss"
-version = "1.1.2"
+version = "1.2.0"
 
 [lib]
 path = "src/lib.rs"
@@ -14,27 +14,27 @@ path = "src/main.rs"
 [dependencies]
 actix-rt = "=2.9.0"
 google-youtube3 = "=5.0.3"
-actix-web = "=4.4.1"
+actix-web = "=4.5.1"
 async-trait = "=0.1.77"
 url = { version="=2.5.0", features = ["serde"]}
 futures = "=0.3.30"
 log = "=0.4.20"
-regex = "=1.10.2"
-reqwest = { version = "=0.11.23", features = ["json"] }
-serde = "=1.0.195"
-serde_json = "=1.0.111"
-tokio = { version = "=1.35.1", features = ["macros", "process"]}
-uuid = { version= "=1.6.1", features = ["v4", "serde"]}
+regex = "=1.10.3"
+reqwest = { version = "=0.11.24", features = ["json"] }
+serde = "=1.0.197"
+serde_json = "=1.0.114"
+tokio = { version = "=1.36.0", features = ["macros", "process"]}
+uuid = { version= "=1.7.0", features = ["v4", "serde"]}
 genawaiter = {version = "=0.99", features = ["futures03"] }
 openssl = { version = "*", features = ["vendored"] } #this is here just to make cross-compiling work during github actions
 rss = { version = "=2.0", features = ["serde"] }
 eyre = "=0.6"
 simple_logger = "=4.3"
 redis = { version = "=0.24", features = ["tokio-comp"] }
 mime = "=0.3.17"
-cached = { version = "=0.47.0", features = ["redis_tokio"] }
+cached = { version = "=0.49.2", features = ["redis_tokio"] }
 iso8601-duration = "=0.2.0"
-chrono = "=0.4.31"
+chrono = "=0.4.34"
 feed-rs = "=1.4.0"
 
 [dev-dependencies]

Dockerfile

@@ -1,7 +1,7 @@
 # by using --platform=$BUILDPLATFORM we force the build step 
 # to always run on the native architecture of the build machine
 # making the build time shorter
-FROM --platform=$BUILDPLATFORM rust:1.74 as builder
+FROM --platform=$BUILDPLATFORM rust:1.75 as builder
 
 ARG BUILDPLATFORM
 ARG TARGETPLATFORM

README.md

@@ -84,7 +84,7 @@ run this inside the folder with `docker-compose.yml`
 
 `sudo docker compose pull && sudo docker compose up -d`
 
-then run this to delete the old version form your system (note: this will also delete any other unused image you have)
+then run this to delete the old version from your system (note: this will also delete any other unused image you have)
 
 `sudo docker system prune`
 

docker-compose.yml

@@ -18,6 +18,10 @@ services:
     #change "latest" to "X.X.X" to pin a version es: "1.0.4" will force the image to use to version 1.0.4, if you do please watch the repo for updates (tutorial in README.md)
     #change "latest" to "beta" if you want to test yet unreleased fixes/features (expect bugs and broken builds from time to time)
     image: madiele/vod2pod-rss:latest 
+    # uncomment to build vod2pod from scratch, only do this if your architecture is not supported
+    #build: 
+    #  dockerfile: ./Dockerfile
+    #  context: https://github.com/madiele/vod2pod-rss.git
     depends_on:
       - redis
     restart: unless-stopped

src/configs/mod.rs

@@ -15,6 +15,7 @@ pub enum ConfName {
     RedisUrl,
     Mp3Bitrate,
     YoutubeApiKey,
+    YouutbeMaxResults,
     TwitchClientId,
     TwitchSecretKey,
     TranscodingEnabled,
@@ -102,6 +103,9 @@ impl Conf for EnvConf {
             ConfName::PeerTubeValidHosts => {
                 Ok(std::env::var("PEERTUBE_VALID_DOMAINS").unwrap_or_else(|_| "".to_string()))
             }
+            ConfName::YouutbeMaxResults => {
+                Ok(std::env::var("YOUTUBE_MAX_RESULTS").unwrap_or_else(|_| "300".to_string()))
+            }
         }
     }
 }

src/provider/twitch.rs

@@ -56,7 +56,7 @@ impl MediaProvider for TwitchProvider {
             .data;
 
         let channel = channels
-            .get(0)
+            .first()
             .ok_or_else(|| eyre::eyre!("No twitch user found"))?;
 
         debug!("fetched twitch channel: {:?}", channel);

src/provider/youtube.rs

@@ -160,7 +160,9 @@ async fn fetch_from_api(id: IdType, api_key: String) -> eyre::Result<(Channel, V
 
             let rss_channel = build_channel_from_playlist(playlist);
 
-            let items = fetch_playlist_items(&playlist_id, &api_key).await?;
+            let max_fetched_items: usize =
+                conf().get(ConfName::YouutbeMaxResults).unwrap().parse()?;
+            let items = fetch_playlist_items(&playlist_id, &api_key, max_fetched_items).await?;
 
             let duration_map = create_duration_url_map(&items, &api_key).await?;
 
@@ -183,7 +185,9 @@ async fn fetch_from_api(id: IdType, api_key: String) -> eyre::Result<(Channel, V
 
             let rss_channel = build_channel_from_yt_channel(channel);
 
-            let items = fetch_playlist_items(&upload_playlist, &api_key).await?;
+            let max_fetched_items: usize =
+                conf().get(ConfName::YouutbeMaxResults).unwrap().parse()?;
+            let items = fetch_playlist_items(&upload_playlist, &api_key, max_fetched_items).await?;
 
             let duration_map = create_duration_url_map(&items, &api_key).await?;
 
@@ -355,37 +359,51 @@ fn build_channel_items_from_playlist(
 async fn fetch_playlist_items(
     playlist_id: &String,
     api_key: &String,
+    max_fetched_items: usize,
 ) -> eyre::Result<Vec<PlaylistItem>> {
     let hub = get_youtube_hub();
-    let max_consecutive_requests = 5;
-    let mut fetched_playlist_items: Vec<PlaylistItem> =
-        Vec::with_capacity(max_consecutive_requests * 50);
+    let max_consecutive_requests = (max_fetched_items / 50) + 1;
+    let mut fetched_playlist_items: Vec<PlaylistItem> = Vec::with_capacity(max_fetched_items);
     let mut request_count = 0;
     let mut next_page_token: Option<String> = None;
     debug!("fetching items from playlist {}", playlist_id);
     loop {
+        let remaining_items = max_fetched_items - fetched_playlist_items.len();
+        let items_to_fetch = if remaining_items > 50 {
+            50
+        } else {
+            remaining_items
+        };
+
         let mut playlist_items_request = hub
             .playlist_items()
             .list(&vec!["snippet".into()])
             .playlist_id(playlist_id)
             .param("key", api_key)
-            .max_results(50);
+            .max_results(items_to_fetch.try_into()?);
+
         if let Some(ref next_page_token) = next_page_token {
             playlist_items_request = playlist_items_request.page_token(next_page_token.as_str());
         }
+
         let response = playlist_items_request.doit().await?;
 
-        fetched_playlist_items.extend(response.1.items.ok_or(eyre!("playlist has no items"))?);
+        fetched_playlist_items.extend(
+            response
+                .1
+                .items
+                .ok_or(eyre!("playlist object has no items field"))?,
+        );
         next_page_token = response.1.next_page_token;
 
-        request_count += 1;
-        if next_page_token.is_none() || request_count > max_consecutive_requests {
+        if next_page_token.is_none() || request_count == max_consecutive_requests {
             info!(
-                "fetched {} items, channel too large, stopping",
+                "fetched {} items, max items reached or no more items to fetch",
                 fetched_playlist_items.len()
             );
             break;
         }
+        request_count += 1;
     }
     info!(
         "fetched {} items, in {} requests",
@@ -671,20 +689,71 @@ mod tests {
 
     #[tokio::test]
     async fn test_build_items_for_playlist_requires_api_key() {
-        let id = "PLJmimp-uZX42T7ONp1FLXQDJrRxZ-_1Ct".to_string();
+        let id = "UUXuqSBlHAE6Xw-yeJA0Tunw".to_string();
         let api_key = conf().get(ConfName::YoutubeApiKey).unwrap();
 
         let playlist = fetch_playlist(id, &api_key).await.unwrap();
 
         println!("{:?}", &playlist.clone().id.unwrap().clone());
-        let items = fetch_playlist_items(&playlist.id.unwrap(), &api_key)
+        let items = fetch_playlist_items(&playlist.id.unwrap(), &api_key, 300)
             .await
             .unwrap();
 
         println!("{:?}", items);
         assert!(!items.is_empty())
     }
 
+    #[tokio::test]
+    async fn test_less_than_50_items_requires_api_key() {
+        let id = "UUXuqSBlHAE6Xw-yeJA0Tunw".to_string();
+        let api_key = conf().get(ConfName::YoutubeApiKey).unwrap();
+
+        let playlist = fetch_playlist(id, &api_key).await.unwrap();
+
+        println!("{:?}", &playlist.clone().id.unwrap().clone());
+        let items = fetch_playlist_items(&playlist.id.unwrap(), &api_key, 13)
+            .await
+            .unwrap();
+
+        println!("{:?}", items);
+        assert!(!items.is_empty());
+        assert_eq!(items.len(), 13)
+    }
+
+    #[tokio::test]
+    async fn test_less_than_300_items_requires_api_key() {
+        let id = "UUXuqSBlHAE6Xw-yeJA0Tunw".to_string();
+        let api_key = conf().get(ConfName::YoutubeApiKey).unwrap();
+
+        let playlist = fetch_playlist(id, &api_key).await.unwrap();
+
+        println!("{:?}", &playlist.clone().id.unwrap().clone());
+        let items = fetch_playlist_items(&playlist.id.unwrap(), &api_key, 50)
+            .await
+            .unwrap();
+
+        println!("{:?}", items);
+        assert!(!items.is_empty());
+        assert_eq!(items.len(), 50)
+    }
+
+    #[tokio::test]
+    async fn test_more_than_300_items_requires_api_key() {
+        let id = "UUXuqSBlHAE6Xw-yeJA0Tunw".to_string();
+        let api_key = conf().get(ConfName::YoutubeApiKey).unwrap();
+
+        let playlist = fetch_playlist(id, &api_key).await.unwrap();
+
+        println!("{:?}", &playlist.clone().id.unwrap().clone());
+        let items = fetch_playlist_items(&playlist.id.unwrap(), &api_key, 600)
+            .await
+            .unwrap();
+
+        println!("{:?}", items);
+        assert!(!items.is_empty());
+        assert_eq!(items.len(), 600)
+    }
+
     #[test(tokio::test)]
     async fn test_build_channel_for_playlist_requires_api_key() {
         let id = "PLJmimp-uZX42T7ONp1FLXQDJrRxZ-_1Ct".to_string();

src/rss_transcodizer/mod.rs

@@ -5,6 +5,7 @@ use std::time::Duration;
 use eyre::eyre;
 use log::debug;
 use reqwest::Url;
+use rss::extension::itunes::ITunesCategory;
 use rss::Channel;
 use rss::{Enclosure, Item};
 
@@ -16,6 +17,11 @@ pub fn inject_vod2pod_customizations(
 ) -> eyre::Result<String> {
     let mut injected_feed = Channel::read_from(rss_body.as_bytes())?;
     injected_feed.set_generator(Some("generated by vod2pod-rss".to_string()));
+    if let Some(ref mut itunes) = injected_feed.itunes_ext {
+        let mut default_category = ITunesCategory::default();
+        default_category.set_text("Technology");
+        itunes.set_categories(vec![default_category]);
+    }
     let mut namespaces = BTreeMap::new();
     namespaces.insert(
         "rss".to_string(),
@@ -25,7 +31,12 @@ pub fn inject_vod2pod_customizations(
         "itunes".to_string(),
         "http://www.itunes.com/dtds/podcast-1.0.dtd".to_string(),
     );
+    namespaces.insert(
+        "content".to_string(),
+        "http://purl.org/rss/1.0/modules/content/".to_string(),
+    );
     injected_feed.set_namespaces(namespaces);
+    injected_feed.set_language("en-US".to_string());
     injected_feed
         .items_mut()
         .iter_mut()

src/server/mod.rs

@@ -1,6 +1,8 @@
 use std::{collections::HashMap, net::TcpListener, time::Instant};
 
-use actix_web::{dev::Server, guard, middleware, web, App, HttpRequest, HttpResponse, HttpServer};
+use actix_web::{
+    dev::Server, guard, http, middleware, web, App, HttpRequest, HttpResponse, HttpServer,
+};
 use log::{debug, error, info, warn};
 use regex::Regex;
 use serde::Deserialize;
@@ -23,12 +25,21 @@ pub fn spawn_server(listener: TcpListener) -> eyre::Result<Server> {
             .service(
                 web::scope(&root)
                     .service(
-                        web::resource("transcode_media/to_mp3")
+                        web::resource("transcode_media/to.mp3")
                             .name("transcode_mp3")
-                            .guard(guard::Get())
+                            .guard(guard::Any(guard::Get()).or(guard::Head()))
+                            .to(transcode_to_mp3),
+                    )
+                    .service(
+                        //this is an old URL used in old vod2pod versions that did not work with
+                        //itunes kept for backwards compatiility
+                        web::resource("transcode_media/to_mp3")
+                            .name("transcode_mp3_obsolete")
+                            .guard(guard::Any(guard::Get()).or(guard::Head()))
                             .to(transcode_to_mp3),
                     )
                     .route("transcodize_rss", web::get().to(transcodize_rss))
+                    .route("transcodize_rss", web::head().to(transcodize_rss))
                     .route("health", web::get().to(health))
                     .route("/", web::get().to(index))
                     .route("", web::get().to(index)),
@@ -63,6 +74,10 @@ async fn transcodize_rss(
     req: HttpRequest,
     query: web::Query<HashMap<String, String>>,
 ) -> HttpResponse {
+    if req.method() == http::Method::HEAD {
+        return HttpResponse::Ok().finish();
+    }
+
     let start_time = Instant::now();
 
     let should_transcode = match conf().get(ConfName::TranscodingEnabled) {
@@ -258,6 +273,17 @@ async fn transcode_to_mp3(req: HttpRequest, query: web::Query<TranscodizeQuery>)
     };
     debug!("seconds: {duration_secs}, bitrate: {bitrate}");
 
+    if req.method() == http::Method::HEAD {
+        return HttpResponse::Ok()
+            .insert_header(("Accept-Ranges", "bytes"))
+            .insert_header((
+                "Content-Range",
+                format!("bytes {start_bytes}-{end_bytes}/{total_streamable_bytes}"),
+            ))
+            .content_type(codec.get_mime_type_str())
+            .finish();
+    }
+
     match Transcoder::new(&ffmpeg_paramenters).await {
         Ok(transcoder) => {
             let stream = transcoder.get_transcode_stream();