feat: burn danmakus and subtitles together

.gitignore

@@ -391,4 +391,6 @@ cookies.json
 
 # The model files exceeds the size limit 100MB of github, so download them separately.
 src/subtitle/models/base.pt
-src/subtitle/models/small.pt
+src/subtitle/models/small.pt
+
+test/

README.md

@@ -45,9 +45,8 @@ graph TD
         videoFolder[(Video 文件夹)]<--间隔两分钟扫描一次-->startScan(启动扫描 Video 文件夹)
         startScan  --判断是否有弹幕-->ifDanmaku{判断}
         ifDanmaku -->|有弹幕| DanmakuFactory[DanmakuFactory]
-        ifDanmaku -->|无弹幕| ffmpeg[ffmpeg]
-        DanmakuFactory[DanmakuFactory] --自动转换弹幕--> ffmpeg[ffmpeg]
-        ffmpeg[ffmpeg] --压制视频--> whisper[whisperASR模型]
+        ifDanmaku -->|无弹幕| whisper[whisperASR模型]
+        DanmakuFactory[DanmakuFactory] --自动转换弹幕--> whisper[whisperASR模型]
         whisper[whisperASR模型] --生成字幕--> ffmpeg1[ffmpeg]
         ffmpeg1[ffmpeg] --渲染字幕 --> uploadQueue[(上传队列)]
 

src/burnAndMerge.sh

@@ -19,13 +19,13 @@ while read -r line; do
     outputFile=$(echo "$line" | sed 's/_\([0-9]\{4\}\)\([0-9]\{2\}\)\([0-9]\{2\}\)/_\1-\2-\3/')
 
     # Convert the danmaku files
-    xmlFile=${line%.mp4}.xml
-    assFile=${line%.mp4}.ass
-    if [ -f "$xmlFile" ]; then
-        python $BILIVE_PATH/src/utils/adjustPrice.py $xmlFile
-        $BILIVE_PATH/src/utils/DanmakuFactory -o "$assFile" -i "$xmlFile" --msgboxfontsize 30 --msgboxsize 400x1000 --ignore-warnings
-        echo "==================== generated $assFile ===================="
-        export ASS_PATH="$assFile"
+    xmlPath=${line%.mp4}.xml
+    assPath=${line%.mp4}.ass
+    if [ -f "$xmlPath" ]; then
+        python $BILIVE_PATH/src/utils/adjustPrice.py $xmlPath
+        $BILIVE_PATH/src/utils/DanmakuFactory -o "$assPath" -i "$xmlPath" --msgboxfontsize 30 --msgboxsize 400x1000 --ignore-warnings
+        echo "==================== generated $assPath ===================="
+        export ASS_PATH="$assPath"
         python3 $BILIVE_PATH/src/utils/removeEmojis.py >> $BILIVE_PATH/logs/removeEmojis.log 2>&1
     fi
 
@@ -38,42 +38,43 @@ while read -r line; do
     echo "==================== create tmp folder $tmpDir ===================="
     fi
 
+    # Generate the srt file via whisper model
+    python $BILIVE_PATH/src/subtitle/generate.py $line > $BILIVE_PATH/logs/burningLog/whisper-$(date +%Y%m-%d-%H%M%S).log 2>&1
+    srtPath="${line%.mp4}.srt"
     fileName=$(basename "$outputFile")
     newPath="$tmpDir/$fileName"
     echo "==================== burning $newPath ===================="
     echo "file '$newPath'" >> mergevideo.txt
-    if [ -f "$assFile" ]; then
-        # The Nvidia GPU accelerating version.
-        ffmpeg -hwaccel cuda -c:v h264_cuvid -i "$line" -c:v h264_nvenc -vf "ass=$assFile" "$newPath" -y -nostdin > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m%d%H%M%S).log 2>&1
+    if [ -f "$assPath" ]; then
         # The only cpu version.
-        # ffmpeg -i "$line" -vf "ass=$assFile" -preset ultrafast "$newPath" -y -nostdin  > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m%d%H%M%S).log 2>&1
-    else
+        # ffmpeg -i "$line" -vf "ass=$assPath" -preset ultrafast "$newPath" -y -nostdin  > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m-%d-%H%M%S).log 2>&1
         # The Nvidia GPU accelerating version.
-        ffmpeg -hwaccel cuda -c:v h264_cuvid -i "$line" -c:v h264_nvenc "$newPath" -y -nostdin > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m%d%H%M%S).log 2>&1
+        # ffmpeg -hwaccel cuda -c:v h264_cuvid -i "$line" -c:v h264_nvenc -vf "ass=$assPath" "$newPath" -y -nostdin > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m-%d-%H%M%S).log 2>&1
+        # The Nvidia GPU accelerate subtitles and danmaku burning
+        # If you don't have NVIDIA GPU, please comment the command below directly, and use the cpu version above.
+        ffmpeg -hwaccel cuda -c:v h264_cuvid -i "$line" -c:v h264_nvenc -vf "subtitles=$srtPath,subtitles=$assPath" "$newPath" -y -nostdin > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m-%d-%H%M%S).log 2>&1
+    else
         # The only cpu version.
-        # ffmpeg -i "$line" -vf -preset ultrafast "$newPath" -y -nostdin  > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m%d%H%M%S).log 2>&1
+        # ffmpeg -i "$line" -vf -preset ultrafast "$newPath" -y -nostdin  > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m-%d-%H%M%S).log 2>&1
+        # The Nvidia GPU accelerating version.
+        ffmpeg -hwaccel cuda -c:v h264_cuvid -i "$line" -c:v h264_nvenc -vf "subtitles=$srtPath" "$newPath" -y -nostdin > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m-%d-%H%M%S).log 2>&1
     fi
 
     # Delete the related items of videos
     rm ${line%.mp4}.*
+    # mv $line ${line%.mp4}
 done < ./src/sameSegments.txt
 
 rm $BILIVE_PATH/src/sameSegments.txt
 # merge the videos
 echo "==================== merge starts ===================="
 # echo "ffmpeg -f concat -i mergevideo.txt -c copy $firstOutputFile"
-ffmpeg -f concat -safe 0 -i mergevideo.txt -use_wallclock_as_timestamps 1 -c copy $firstOutputFile > $BILIVE_PATH/logs/mergeLog/merge-$(date +%Y%m%d%H%M%S).log 2>&1
+ffmpeg -f concat -safe 0 -i mergevideo.txt -use_wallclock_as_timestamps 1 -c copy $firstOutputFile > $BILIVE_PATH/logs/mergeLog/merge-$(date +%Y%m-%d-%H%M%S).log 2>&1
 
 # delete useless videos and lists
 rm -r $tmpDir
 rm mergevideo.txt
-
-python $BILIVE_PATH/src/subtitle/generate.py $firstOutputFile > $BILIVE_PATH/logs/burningLog/subtitlesGenerate-$(date +%Y%m%d%H%M%S).log 2>&1
-srtPath=${firstOutputFile%.*}".srt"
-videoUploadPath=${firstOutputFile%.*}"-s.mp4"
-ffmpeg -hwaccel cuda -c:v h264_cuvid -i "$firstOutputFile" -c:v h264_nvenc -vf "subtitles=$srtPath" "$videoUploadPath" -y -nostdin > $BILIVE_PATH/logs/burningLog/subtitlesRender-$(date +%Y%m%d%H%M%S).log 2>&1
-rm $srtPath
 rm $firstOutputFile
 
 echo "==================== add $videoUploadPath to upload queue ===================="
-echo "$videoUploadPath" >> $BILIVE_PATH/src/uploadProcess/uploadVideoQueue.txt
+echo "$firstOutputFile" >> $BILIVE_PATH/src/uploadProcess/uploadVideoQueue.txt

src/danmakuBurning.sh

@@ -17,18 +17,10 @@ fi
 # obtain full path
 fullPath=$1
 
-# delete json files
-filenameWithoutExt="${fullPath%.*}"
-jsonlPath="${filenameWithoutExt}.jsonl"
-rm $jsonlPath
-
 # use dirname to obtain the path，along with basename to obtain filename.
 path=$(dirname "$fullPath")
 filename=$(basename "$fullPath")
 
-# extract the extonsion name.
-extension="${filename##*.}"
-
 roomid=$(echo "$filename" | cut -d'_' -f1)
 timestamp=$(echo "$filename" | cut -d'_' -f2)
 
@@ -39,47 +31,48 @@ day=${timestamp:6:2}
 hour=${timestamp:9:2}
 
 # format the file name
-formatVideoName="${path}/${roomid}_${year}-${month}-${day}-${hour}.${extension}"
-
-# generate the converted subtitle file name
-assPath="${path}/${roomid}_${year}-${month}-${day}-${hour}.ass"
+formatVideoName="${path}/${roomid}_${year}-${month}-${day}-${hour}.mp4"
 
-# use DanmakuFactory to convert the xml file
-xmlPath="${filenameWithoutExt}.xml"
+# Process the xml file as ass file
+assPath="${fullPath%.*}.ass"
+xmlPath="${fullPath%.*}.xml"
 if [ -f "$xmlPath" ]; then
     python $BILIVE_PATH/src/utils/adjustPrice.py $xmlPath
     $BILIVE_PATH/src/utils/DanmakuFactory -o "$assPath" -i "$xmlPath" --msgboxfontsize 30 --ignore-warnings
-    rm $xmlPath
     echo “danmaku convert success!”
     export ASS_PATH="$assPath"
     python3 $BILIVE_PATH/src/utils/removeEmojis.py >> $BILIVE_PATH/logs/removeEmojis.log 2>&1
 fi
 
+# Generate the srt file via whisper model
+# If you don't have NVIDIA GPU, please comment the command below directly.
+python $BILIVE_PATH/src/subtitle/generate.py $fullPath > $BILIVE_PATH/logs/burningLog/whisper-$(date +%Y%m-%d-%H%M%S).log 2>&1
+srtPath="${fullPath%.*}.srt"
+
 # Burn danmaku into video.
 if [ -f "$assPath" ]; then
     # The only cpu version
-    # ffmpeg -y -i $fullPath -vf ass=$assPath -preset ultrafast $formatVideoName > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m%d%H%M%S).log 2>&1
+    # ffmpeg -y -i $fullPath -vf ass=$assPath -preset ultrafast $formatVideoName > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m-%d-%H%M%S).log 2>&1
     # The Nvidia GPU accelerating version
-    ffmpeg -y -hwaccel cuda -c:v h264_cuvid -i $fullPath -c:v h264_nvenc -vf ass=$assPath $formatVideoName > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m%d%H%M%S).log 2>&1
-    rm $assPath
+    # ffmpeg -y -hwaccel cuda -c:v h264_cuvid -i $fullPath -c:v h264_nvenc -vf ass=$assPath $formatVideoName > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m-%d-%H%M%S).log 2>&1
+    # The Nvidia GPU accelerate subtitles and danmaku burning
+    # If you don't have NVIDIA GPU, please comment the command below directly, and use the cpu version above.
+    ffmpeg -y -hwaccel cuda -c:v h264_cuvid -i $fullPath -c:v h264_nvenc -vf "subtitles=$srtPath,subtitles=$assPath" $formatVideoName > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m-%d-%H%M%S).log 2>&1
 else
-    # The only cpu version
-    # ffmpeg -y -i $fullPath -vf -preset ultrafast $formatVideoName > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m%d%H%M%S).log 2>&1
-    # The Nvidia GPU acceleting version
-    ffmpeg -y -hwaccel cuda -c:v h264_cuvid -i $fullPath -c:v h264_nvenc $formatVideoName > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m%d%H%M%S).log 2>&1
+    # If the ass file is not found, only burn the subtitles into video.
+    # Nonetheless, If you don't have Nvidia GPU to burn the subtitles, please comment the command below directly.
+    # The Nvidia GPU accelerate subtitles burning
+    ffmpeg -y -hwaccel cuda -c:v h264_cuvid -i $fullPath -c:v h264_nvenc -vf "subtitles=$srtPath" $formatVideoName > $BILIVE_PATH/logs/burningLog/burn-$(date +%Y%m-%d-%H%M%S).log 2>&1
 fi
 
 echo "ffmpeg successfully complete!"
 
 # Delete the original video.
 rm $fullPath
-
-python $BILIVE_PATH/src/subtitle/generate.py $formatVideoName > $BILIVE_PATH/logs/burningLog/subtitlesGenerate-$(date +%Y%m%d%H%M%S).log 2>&1
-srtPath=${formatVideoName%.*}".srt"
-videoUploadPath=${formatVideoName%.*}"-s.mp4"
-ffmpeg -hwaccel cuda -c:v h264_cuvid -i "$formatVideoName" -c:v h264_nvenc -vf "subtitles=$srtPath" "$videoUploadPath" -y -nostdin > $BILIVE_PATH/logs/burningLog/subtitlesRender-$(date +%Y%m%d%H%M%S).log 2>&1
+rm $xmlPath
+rm $assPath
 rm $srtPath
-rm $formatVideoName
+# mv $fullPath ${fullPath%.*}
 
-echo "==================== add $videoUploadPath to upload queue ===================="
-echo "$videoUploadPath" >> $BILIVE_PATH/src/uploadProcess/uploadVideoQueue.txt
+echo "==================== add $formatVideoName to upload queue ===================="
+echo "$formatVideoName" >> $BILIVE_PATH/src/uploadProcess/uploadVideoQueue.txt

src/uploadProcess/uploadVideo.sh

@@ -21,7 +21,7 @@ uploadPath=$1
 # extract the roomid and date information
 fileName=$(basename "$uploadPath")
 # use regular expression to match the parameter
-if [[ $fileName =~ ([0-9]+)_([0-9]{4})-([0-9]{2})-([0-9]{2})-([0-9]{2})-s\.mp4 ]]; then
+if [[ $fileName =~ ([0-9]+)_([0-9]{4})-([0-9]{2})-([0-9]{2})-([0-9]{2})\.mp4 ]]; then
     roomID="${BASH_REMATCH[1]}"
     year="${BASH_REMATCH[2]}"
     month="${BASH_REMATCH[3]}"