Merge pull request #130 from madiele/main

v1.1.0

.cargo/config.toml

@@ -1,2 +1,6 @@
 [net]
 git-fetch-with-cli = true
+[target.armv7-unknown-linux-gnueabihf]
+linker = "arm-linux-gnueabihf-gcc"
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"

.github/workflows/rust-clippy.yml

@@ -0,0 +1,43 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# rust-clippy is a tool that runs a bunch of lints to catch common
+# mistakes in your Rust code and help improve your Rust code.
+# More details at https://github.com/rust-lang/rust-clippy
+# and https://rust-lang.github.io/rust-clippy/
+
+name: clippy
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  rust-clippy-analyze:
+    name: cargo clippy
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      security-events: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install required cargo
+        run: cargo install clippy-sarif sarif-fmt
+
+      - name: Run rust-clippy
+        run:
+          cargo clippy
+          --all-features
+          --message-format=json | clippy-sarif | tee rust-clippy-results.sarif | sarif-fmt
+        continue-on-error: true
+
+      - name: Upload analysis results to GitHub
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: rust-clippy-results.sarif
+          wait-for-processing: true

.github/workflows/rust.yml

@@ -9,11 +9,29 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
+  YT_API_KEY: ${{ secrets.YT_API_KEY }}
+  TWITCH_SECRET: ${{ secrets.TWITCH_SECRET }}
+  TWITCH_CLIENT_ID: ${{ secrets.TWITCH_CLIENT_ID }}
 
 jobs:
-  build:
+  check-secret:
+    runs-on: ubuntu-latest
+    outputs:
+      has_secrets: ${{ steps.my-key-check.outputs.defined }}
+    steps:
+      - name: Check for Secret availability
+        id: my-key-check
+        shell: bash
+        run: |
+          if [ "${{ secrets.TWITCH_CLIENT_ID }}" != '' ]; then
+            echo "defined=true" >> $GITHUB_OUTPUT;
+          else
+            echo "defined=false" >> $GITHUB_OUTPUT;
+          fi
 
+  build:
     runs-on: ubuntu-latest
+    needs: [check-secret]
 
     steps:
       - uses: actions/checkout@v4
@@ -28,5 +46,9 @@ jobs:
         run: python3 -m pip install -U yt-dlp
       - name: Start Redis server
         run: redis-server --daemonize yes
-      - name: Run tests
+      - name: Run tests with secrets
         run: cargo test --verbose
+        if: needs.check-secret.outputs.has_secrets == 'true'
+      - name: Run tests without secrets
+        run: cargo test --verbose -- --skip requires_api_key
+        if: needs.check-secret.outputs.has_secrets == 'false'

CONTRIBUTING.md

@@ -9,19 +9,33 @@ When adding a feature try to also add some test if possible.
 
 ## Pull Request Process
 
-All pull request should point to the main branch and should pass all tests (`make test`)
+All pull request should point to the main branch and should pass all tests (`make test`),
+you are allowed to edit the tests if they stop having a meaning after your changes.
 
 ## How does it work?
 
-vod2pod-rss under the hood is a service that takes an hosted RSS feed (public feed or generated by a service reggistered in it's docker-compose.yml)
-and adapts it by rewriting every media url to point to the trancode endpoint of vod2pod-rss webserver, you can actually give it any
-RSS podcast feed and if ffmpeg can transcode the contained media URLS then it will work (you would need to add the domain to the whitelist, check the README for more info). 
+vod2pod-rss is composed of 3 different phases:
 
-If that is not enough and you want to add a new media provider that need some API calls to work just check the provider folder
-and implement the MediaProvider trait (more info are in the module), then feel free to open a PR.
+* feed generation
+* rewriting of the generated feed to inject the transcoded enabled urls
+* live transcoding of the videos with ffmpeg
 
-also check the [podtube](https://github.com/amckee/PodTube) repo if the provider you need is already offered by it, if it does you just need to
-write the provider to point to the right podtube feed urls
+feed generation is done by implementing the `MediaProvider` trait which describes the generation of the raw rss feed, it's very important that
+the provider generated a duration for each entry or seeking will not work.
+
+this is then passed to the feed injection that will prepare the generic rss feed with the necessary links to the trancode endpoint.
+one key rule is that the MediaProvider has no idea that the generated feed is intended for trancoding, this way if you disable trancoding
+you still endup with a valid feed.
+
+once a client makes a request it goes through the transcoding endpoint it will start a live trancode with ffmpeg and return the transcoded stream
+
+optimization: most calls are cached with redis
+
+### How to make a new provider
+
+in the provider folder you will find the list of supported provider, to make a new one
+add a file for your provider, implement the MediaProvider trait (you can copy generic.rs
+to get a boilerplate) and then add it to provider/mod.rs inside the macro call
 
 ## Initial setup
 

Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["madiele92@gmail.com"]
 edition = "2021"
 name = "vod2pod-rss"
-version = "1.0.6"
+version = "1.1.0"
 
 [lib]
 path = "src/lib.rs"
@@ -13,28 +13,29 @@ path = "src/main.rs"
 
 [dependencies]
 actix-rt = "=2.9.0"
+google-youtube3 = "=5.0.3"
 actix-web = "=4.4.0"
 async-trait = "=0.1.74"
-url = { version="*", features = ["serde"]}
-futures = "=0.3.28"
+url = { version="=2.4.1", features = ["serde"]}
+futures = "=0.3.29"
 log = "=0.4.20"
 regex = "=1.10.2"
 reqwest = { version = "=0.11.22", features = ["json"] }
-serde = "=1.0.189"
-serde_json = "=1.0.107"
-tokio = { version = "=1.33.0" , features = ["macros", "process"]}
-uuid = { version= "=1.4.1", features = ["v4", "serde"]}
+serde = "=1.0.192"
+serde_json = "=1.0.108"
+tokio = { version = "=1.34.0", features = ["macros", "process"]}
+uuid = { version= "=1.6.1", features = ["v4", "serde"]}
 genawaiter = {version = "=0.99", features = ["futures03"] }
+openssl = { version = "*", features = ["vendored"] } #this is here just to make cross-compiling work during github actions
 rss = { version = "=2.0", features = ["serde"] }
 eyre = "=0.6"
 simple_logger = "=4.2"
 redis = { version = "=0.23", features = ["tokio-comp"] }
-feed-rs = "=1.3.0"
-md5 = "=0.7.0"
 mime = "=0.3.17"
-cached = { version = "=0.46.0", features = ["redis_tokio"] }
+cached = { version = "=0.46.1", features = ["redis_tokio"] }
 iso8601-duration = "=0.2.0"
 chrono = "=0.4.31"
+feed-rs = "=1.3.0"
 
 [dev-dependencies]
 temp-env ={ version = "=0.3.6", features = ["async_closure"] }

Dockerfile

@@ -1,37 +1,66 @@
-FROM rust:1.70 as builder
+# by using --platform=$BUILDPLATFORM we force the build step 
+# to always run on the native architecture of the build machine
+# making the build time shorter
+FROM --platform=$BUILDPLATFORM rust:1.74 as builder
+
+ARG BUILDPLATFORM
+ARG TARGETPLATFORM
+
+#find the right build target for rust
+RUN if [ "$TARGETPLATFORM" = "linux/arm/v7" ]; then \
+        export RUST_TARGET_PLATFORM=armv7-unknown-linux-gnueabihf; \
+    elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
+        export RUST_TARGET_PLATFORM=aarch64-unknown-linux-gnu; \
+    elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
+        export RUST_TARGET_PLATFORM=x86_64-unknown-linux-gnu; \
+    else \
+        export RUST_TARGET_PLATFORM=$(rustup target list --installed | head -n 1); \
+    fi; \
+    echo "choosen rust target: $RUST_TARGET_PLATFORM" ;\
+    echo $RUST_TARGET_PLATFORM > /rust_platform.txt
+
+RUN echo "I am running on $BUILDPLATFORM, building for $TARGETPLATFORM, rust target is $(cat /rust_platform.txt)"
+
+RUN if echo $TARGETPLATFORM | grep -q 'arm'; then \
+        echo 'Installing packages for ARM platforms...'; \
+        apt-get update && apt-get install  build-essential gcc gcc-arm* gcc-aarch* -y && apt-get clean; \
+        echo 'gcc-arm* packages installed and cache cleaned.'; \
+    fi
+
+RUN rustup target add $(cat /rust_platform.txt) 
 
 RUN cd /tmp && USER=root cargo new --bin vod2pod
+
 WORKDIR /tmp/vod2pod
+
 COPY Cargo.toml ./
+#trick to use github action cache, check the action folder for more info
 RUN sed '/\[dev-dependencies\]/,/^$/d' Cargo.toml > Cargo.toml.tmp && mv Cargo.toml.tmp Cargo.toml
 
 RUN cargo fetch
-RUN cargo install cargo-build-deps
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends ffmpeg clang libavformat-dev libavfilter-dev libavcodec-dev libavdevice-dev libavutil-dev libpostproc-dev libswresample-dev libswscale-dev && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-#workaround for https://github.com/rust-lang/cargo/issues/8719
-ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 
-RUN cargo build-deps --release
-COPY src /tmp/vod2pod/src
-
-#trick to use github action cache, check the action folder for more info
+COPY .cargo/ ./.cargo/
+COPY src ./src
 COPY set_version.sh version.txt* ./
 COPY templates/ ./templates/
+
 RUN sh set_version.sh
 
-RUN cargo build --release
+RUN echo "final size of vod2pod:\n $(ls -lah /tmp/vod2pod/target/*/release/app)"
+
+RUN cargo build --release --target "$(cat /rust_platform.txt)"
 
 #----------
-FROM debian:bullseye-slim
+#this step will always run on the target architecture,
+#so the build driver will need to be able to support runtime commands on it (es: using QEMU)  
+FROM --platform=$TARGETPLATFORM debian:bookworm-slim as app
 
-#install ffmpeg and yt-dlp
+ARG BUILDPLATFORM
 ARG TARGETPLATFORM
+
+RUN echo "I am running on $BUILDPLATFORM, building for $TARGETPLATFORM"
 COPY requirements.txt ./
+#install ffmpeg and yt-dlp
 RUN apt-get update && \
     apt-get install -y --no-install-recommends python3 curl ca-certificates ffmpeg && \
     export YT_DLP_VERSION=$(cat requirements.txt | grep yt-dlp | cut -d "=" -f3 | awk -F. '{printf "%d.%02d.%02d\n", $1, $2, $3}') && \
@@ -42,9 +71,16 @@ RUN apt-get update && \
     apt-get -y clean && \
     rm -rf /var/lib/apt/lists/*
 
-COPY --from=builder /tmp/vod2pod/target/release/app /usr/local/bin/vod2pod
+COPY --from=builder /tmp/vod2pod/target/*/release/app /usr/local/bin/vod2pod
 COPY --from=builder /tmp/vod2pod/templates/ ./templates
 
+RUN if vod2pod --version; then \
+        echo "vod2pod starts correctly"; \
+        exit 0; \
+    else \
+        echo "vod2pod did not start" 1>&2; \
+        exit 1; \
+    fi
 
 EXPOSE 8080
 

Makefile

@@ -9,8 +9,6 @@ grep -q '^TWITCH_CLIENT_ID=' .env || echo 'TWITCH_CLIENT_ID=' >> .env
 grep -q '^RUST_LOG=' .env || echo 'RUST_LOG=DEBUG' >> .env
 grep -q '^MP3_BITRATE=' .env || echo 'MP3_BITRATE=192' >> .env
 grep -q '^SUBFOLDER=' .env || echo 'SUBFOLDER=/' >> .env
-grep -q '^TWITCH_TO_PODCAST_URL=' .env || echo 'TWITCH_TO_PODCAST_URL=localhost:8085' >> .env
-grep -q '^PODTUBE_URL=' .env || echo 'PODTUBE_URL=http://localhost:15000' >> .env
 grep -q '^REDIS_ADDRESS=' .env || echo 'REDIS_ADDRESS=localhost' >> .env
 grep -q '^REDIS_PORT=' .env || echo 'REDIS_PORT=6379' >> .env
 endef

README.md

@@ -99,8 +99,10 @@ You can set the following environment variables:
 
 ## Honorable mentions
 
-* Youtube support is possible thanks to the cool [podtube fork project by amckee](https://github.com/amckee/PodTube) consider dropping him a star.
-* Twitch support is possible thanks to [my fork](https://github.com/madiele/TwitchToPodcastRSS) of [lzeke0's TwitchRSS](https://github.com/lzeke0/TwitchRSS) drop a star to him too!
+This projects were fondamental for the success of vod2pod-rss, originally they handled the feed generation for youtube and twitch, now this is all done by vod2pod-rss internally so they are not used anymore, but were still helpful to get vod2pod-rss up and running fast.
+Those project were:
+* Youtube support was possible thanks to the cool [podtube fork project by amckee](https://github.com/amckee/PodTube) consider dropping him a star.
+* Twitch support was possible thanks to [my fork](https://github.com/madiele/TwitchToPodcastRSS) of [lzeke0's TwitchRSS](https://github.com/lzeke0/TwitchRSS) drop a star to him too!
 
 ## Donations
 

SECURITY.md

@@ -3,7 +3,8 @@
 ## Supported Versions
 | Version | Supported          |
 | ------- | ------------------ |
-| 1.x.x   | :white_check_mark: |
+| 1.1.x   | :white_check_mark: |
+| 1.0.x   | :x: |
 | 0.x.x   | :x: |
 
 ## Reporting a Vulnerability

docker-compose.dev_enviroment.yml

@@ -11,26 +11,6 @@ services:
       # Set your Twitch client ID
       - TWITCH_CLIENT_ID=${TWITCH_CLIENT_ID:-}
 
-  ttprss:
-    extends: api_keys
-    image: madiele/twitch_to_podcast_rss
-    restart: "no"
-    container_name: twitchToPodcastRSS
-    ports:
-      - "8085:80"
-    environment:
-        - TZ=Europe/London
-        - TRANSCODE=False
-
-  podtube:
-    extends: api_keys
-    image: ftawesome/podtube
-    restart: "no"
-    ports:
-      - "15000:15000"
-    environment:
-      - YT_AUTOLOAD_NEWEST_AUDIO=False
-
   redis:
     image: "redis:6.2"
     command: redis-server --save 20 1 --loglevel warning

docker-compose.yml

@@ -5,6 +5,7 @@ services:
     image: alpine
     environment:
     #add your api-keys here (put them after the "=" sign or use an .env file)
+    #(is normal to se this conteiner as stopped soon after deploy)
       # Set your YouTube API key
       - YT_API_KEY=${YT_API_KEY:-}
       # Set your Twitch secret
@@ -14,7 +15,7 @@ services:
 
   vod2pod:
     extends: api_keys
-    image: madiele/vod2pod-rss
+    image: madiele/vod2pod-rss:latest
     depends_on:
       - redis
     restart: unless-stopped
@@ -26,27 +27,9 @@ services:
       - TRANSCODE=true #put to false if you only need feed generation
       - SUBFOLDER=/ #for reverse proxies, ex: "/" -> access the app at mywebsite.com ; "vod2pod" -> access at mywebsite.com/vod2pod
       - RUST_LOG=INFO #set to DEBUG if you are having problems than open a github issue with the logs, use "sudo docker compose logs" to print them
-      - TWITCH_TO_PODCAST_URL=ttprss #don't edit this
-      - PODTUBE_URL=http://podtube:15000 #don't edit this
       - REDIS_ADDRESS=redis #don't edit this
       - REDIS_PORT=6379 #don't edit this
 
-  ttprss: # you can remove this section and save some storage if you don't need twitch
-    extends: api_keys
-    image: madiele/twitch_to_podcast_rss
-    restart: unless-stopped
-    container_name: twitchToPodcastRSS
-    environment:
-        - TZ=Europe/London
-        - TRANSCODE=False #keep this to false, the vod2pod one is the one you should change
-
-  podtube: #remove this section if you don't need improved youtube support
-    extends: api_keys
-    image: ftawesome/podtube
-    restart: unless-stopped
-    environment:
-      - YT_AUTOLOAD_NEWEST_AUDIO=False
-
   redis:
     image: "redis:6.2"
     command: redis-server --save 20 1 --loglevel warning

requirements.txt

@@ -1,2 +1,2 @@
 # this is only used to track the version in the Dockerfile and with depend-a-bot
-yt-dlp==2023.10.13
+yt-dlp==2023.11.16