Add check.latest_videos_count and max_per_user options (#754)

config/default.yaml

@@ -541,7 +541,11 @@ import:
       enabled: false
   video_channel_synchronization:
     enabled: false
-    check_interval: 1 hour
+    max_per_user: 10
+    check:
+      interval: 1 hour
+      # Number of latest published videos to check and to potentially import
+      latest_videos_count: 3
 
 auto_blacklist:
   # New videos automatically blacklisted so moderators can review before publishing

config/dev.yaml

@@ -83,7 +83,10 @@ import:
       enabled: true
   video_channel_synchronization:
     enabled: true
-    check_interval: 5 minutes
+    max_per_user: 10
+    check:
+      interval: 5 minutes
+      latest_videos_count: 3 # Number of latest published videos to check and to potentially import
 
 instance:
   default_nsfw_policy: 'display'

config/production.yaml.example

@@ -549,6 +549,17 @@ import:
       # We recommend to only enable magnet URI/torrent import if you trust your users
       # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
       enabled: false
+  # Add ability for your users to synchronize their channels with external channels, playlists, etc.
+  video_channel_synchronization:
+    # If you enable the channel synchronization, ensure to enable the video import through HTTP (import.videos.http.enabled)
+    enabled: false
+    # The number of synchronizations a user can create
+    max_per_user: 10
+    check:
+      # The check interval during which PeerTube imports the new videos published remotely
+      interval: 1 hour
+      # Number of latest published videos to check and to potentially import
+      latest_videos_count: 3
 
 auto_blacklist:
   # New videos automatically blacklisted so moderators can review before publishing

server/controllers/api/config.ts

@@ -273,7 +273,8 @@ function customConfig (): CustomConfig {
         }
       },
       videoChannelSynchronization: {
-        enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED
+        enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED,
+        maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER
       }
     },
     trending: {

server/helpers/youtube-dl/youtube-dl-cli.ts

@@ -123,12 +123,12 @@ export class YoutubeDLCLI {
 
   getChannelInfo (options: {
     channelUrl: string
-    lastVideosCount?: number
+    latestVideosCount?: number
     processOptions: execa.NodeOptions
   }) {
     const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
-    if (options.lastVideosCount !== undefined) {
-      additionalYoutubeDLArgs.push('--playlist-end', options.lastVideosCount.toString())
+    if (options.latestVideosCount !== undefined) {
+      additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
     }
     return this.getInfo({
       url: options.channelUrl,

server/initializers/checker-before-init.ts

@@ -32,6 +32,8 @@ function checkMissedConfig () {
     'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',
     'transcoding.resolutions.2160p', 'video_studio.enabled',
     'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
+    'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
+    'import.video_channel_synchronization.check.interval', 'import.video_channel_synchronization.check.latest_videos_count',
     'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
     'client.videos.miniature.display_author_avatar',
     'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',

server/initializers/config.ts

@@ -398,7 +398,13 @@ const CONFIG = {
     },
     VIDEO_CHANNEL_SYNCHRONIZATION: {
       get ENABLED () { return config.get<boolean>('import.video_channel_synchronization.enabled') },
-      get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) }
+      get MAX_PER_USER () { return config.get<number>('import.video_channel_synchronization.max_per_user') },
+      CHECK: {
+        get INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check.interval')) },
+        get LATEST_VIDEOS_COUNT () {
+          return parseDurationToMs(config.get<number>('import.video_channel_synchronization.check.latest_videos_count'))
+        }
+      }
     }
   },
   AUTO_BLACKLIST: {

server/initializers/constants.ts

@@ -246,7 +246,7 @@ const SCHEDULER_INTERVALS_MS = {
   REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
   UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
   REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour
-  CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL
+  CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK.INTERVAL
 }
 
 // ---------------------------------------------------------------------------
@@ -461,8 +461,6 @@ const VIDEO_PRIVACIES: { [ id in VideoPrivacy ]: string } = {
   [VideoPrivacy.INTERNAL]: 'Internal'
 }
 
-const VIDEO_CHANNEL_MAX_SYNC = 3
-
 const VIDEO_STATES: { [ id in VideoState ]: string } = {
   [VideoState.PUBLISHED]: 'Published',
   [VideoState.TO_TRANSCODE]: 'To transcode',
@@ -993,7 +991,6 @@ export {
   VIDEO_LANGUAGES,
   VIDEO_PRIVACIES,
   VIDEO_LICENCES,
-  VIDEO_CHANNEL_MAX_SYNC,
   VIDEO_STATES,
   WORKER_THREADS,
   VIDEO_RATE_TYPES,

server/lib/schedulers/video-channel-sync-latest-scheduler.ts

@@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config'
 import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
 import { MChannelSyncChannel } from '@server/types/models'
 import { VideoChannelSyncState } from '@shared/models'
-import { SCHEDULER_INTERVALS_MS, VIDEO_CHANNEL_MAX_SYNC } from '../../initializers/constants'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
 import { synchronizeChannel } from '../video-import-channel'
 import { AbstractScheduler } from './abstract-scheduler'
 
@@ -35,7 +35,7 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
         const { errors, successes, alreadyImported } = await synchronizeChannel(sync.VideoChannel, sync.externalChannelUrl, {
           youtubeDL,
           secondsToWait: 5,
-          lastVideosCount: VIDEO_CHANNEL_MAX_SYNC,
+          latestVideosCount: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK.LATEST_VIDEOS_COUNT,
           onlyAfter: syncCreationDate
         })
         if (errors > 0) {

server/lib/video-import-channel.ts

@@ -21,7 +21,7 @@ const processOptions = {
 export type SynchronizeChannelOptions = {
   youtubeDL: YoutubeDLCLI
   secondsToWait: number
-  lastVideosCount?: number
+  latestVideosCount?: number
   onlyAfter?: Date
 }
 
@@ -32,11 +32,11 @@ function formatDateForYoutubeDl (date: Date) {
 export async function synchronizeChannel (
   channel: MChannelAccountDefault,
   externalChannelUrl: string,
-  { youtubeDL, secondsToWait, lastVideosCount, onlyAfter }: SynchronizeChannelOptions
+  { youtubeDL, secondsToWait, latestVideosCount, onlyAfter }: SynchronizeChannelOptions
 ): Promise<ChannelSyncInfo> {
   const user = await UserModel.loadByChannelActorId(channel.actorId)
   const channelInfo = await youtubeDL.getChannelInfo({
-    lastVideosCount,
+    latestVideosCount,
     channelUrl: externalChannelUrl,
     processOptions
   })
@@ -62,7 +62,6 @@ export async function synchronizeChannel (
 
   for (const targetUrl of targetUrls) {
     try {
-      // TODO retry pour l'import d'une chaîne ?
       if (!await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
         const { job } = await addYoutubeDLImport({
           user,

server/middlewares/validators/videos/video-channel-sync.ts

@@ -31,6 +31,14 @@ export const videoChannelSyncValidator = [
       return
     }
 
+    const count = await VideoChannelSyncModel.countByAccount(res.locals.videoChannel.accountId)
+    if (count >= CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER) {
+      res.fail({
+        message: `You cannot create more than ${CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER} channel synchronizations`
+      })
+      return false
+    }
+
     return next()
   }
 ]

server/models/video/video-channel-sync.ts

@@ -107,6 +107,22 @@ export class VideoChannelSyncModel extends Model<Partial<AttributesOnly<VideoCha
     ]).then(([ total, data ]) => ({ total, data }))
   }
 
+  static countByAccount (accountId: number) {
+    const query = {
+      include: [
+        {
+          model: VideoChannelModel.unscoped(),
+          required: true,
+          where: {
+            accountId
+          }
+        }
+      ]
+    }
+
+    return VideoChannelSyncModel.unscoped().count(query)
+  }
+
   static loadWithAccount (id: number) {
     return this.unscoped().findByPk(id, {
       include: [ {

server/tests/api/check-params/config.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
 import 'mocha'
-import { omit } from 'lodash'
+import { chain, omit } from 'lodash'
 import {
   cleanupTests,
   createSingleServer,
@@ -162,7 +162,8 @@ describe('Test config API validators', function () {
         }
       },
       videoChannelSynchronization: {
-        enabled: false
+        enabled: false,
+        maxPerUser: 10
       }
     },
     trending: {
@@ -348,20 +349,11 @@ describe('Test config API validators', function () {
     })
 
     it('Should fail with a disabled http upload & enabled sync', async function () {
-      const newUpdateParams: CustomConfig = {
-        ...updateParams,
-        import: {
-          videos: {
-            ...updateParams.import.videos,
-            http: {
-              enabled: false
-            }
-          },
-          videoChannelSynchronization: {
-            enabled: true
-          }
-        }
-      }
+      const newUpdateParams: CustomConfig = chain(updateParams)
+        .cloneDeep()
+        .update('import.videos.http.enabled', () => false)
+        .update('import.videoChannelSynchronization.enabled', () => true)
+        .value()
       await makePutBodyRequest({
         url: server.url,
         path,

server/tests/api/check-params/video-channel-syncs.ts

@@ -8,6 +8,7 @@ import {
   setAccessTokensToServers,
   setDefaultVideoChannel
 } from '@shared/server-commands'
+import { chain } from 'lodash'
 import 'mocha'
 
 describe('Test video channel sync API validator', () => {
@@ -33,6 +34,21 @@ describe('Test video channel sync API validator', () => {
     }
   }
 
+  async function withMaxSyncsPerUser<T> (maxSync: number, callback: () => Promise<T>): Promise<void> {
+    const origConfig = await server.config.getCustomConfig()
+    const newConfig = chain(origConfig).cloneDeep().update('import.videoChannelSynchronization.maxPerUser', () => 1).value()
+    await server.config.updateCustomConfig({
+      newCustomConfig: newConfig
+    })
+    try {
+      await callback()
+    } finally {
+      await server.config.updateCustomConfig({
+        newCustomConfig: origConfig
+      })
+    }
+  }
+
   before(async function () {
     this.timeout(30_000)
 
@@ -159,6 +175,19 @@ describe('Test video channel sync API validator', () => {
       })
       rootChannelSyncId = res.id
     })
+
+    it('Should fail when the user exceeds allowed number of synchronizations', async function () {
+      await withMaxSyncsPerUser(1, async () => {
+        await command.create({
+          token: server.accessToken,
+          attributes: {
+            ...baseCorrectParams,
+            videoChannelId: userInfo.channelId
+          },
+          expectedStatus: HttpStatusCode.BAD_REQUEST_400
+        })
+      })
+    })
   })
 
   describe('When listing my video imports', function () {

server/tests/api/server/config.ts

@@ -364,7 +364,8 @@ const newCustomConfig: CustomConfig = {
       }
     },
     videoChannelSynchronization: {
-      enabled: false
+      enabled: false,
+      maxPerUser: 10
     }
   },
   trending: {

shared/models/server/custom-config.model.ts

@@ -164,6 +164,7 @@ export interface CustomConfig {
     }
     videoChannelSynchronization: {
       enabled: boolean
+      maxPerUser: number
     }
   }
 

shared/server-commands/server/config-command.ts

@@ -384,7 +384,8 @@ export class ConfigCommand extends AbstractCommand {
           }
         },
         videoChannelSynchronization: {
-          enabled: false
+          enabled: false,
+          maxPerUser: 10
         }
       },
       trending: {