Tests finished + convertion of job to scheduler (#754)

server.ts

@@ -139,6 +139,7 @@ import { VideoViewsManager } from '@server/lib/views/video-views-manager'
 import { isTestOrDevInstance } from './server/helpers/core-utils'
 import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
 import { ApplicationModel } from '@server/models/application/application'
+import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
 
 // ----------- Command line -----------
 
@@ -314,6 +315,7 @@ async function startApplication () {
   PeerTubeVersionCheckScheduler.Instance.enable()
   AutoFollowIndexInstances.Instance.enable()
   RemoveDanglingResumableUploadsScheduler.Instance.enable()
+  VideoChannelSyncLatestScheduler.Instance.enable()
   VideoViewsBufferScheduler.Instance.enable()
   GeoIPUpdateScheduler.Instance.enable()
   OpenTelemetryMetrics.Instance.registerMetrics()

server/controllers/api/video-channel-sync.ts

@@ -50,7 +50,7 @@ async function createVideoChannelSync (req: express.Request, res: express.Respon
   const syncCreated = new VideoChannelSyncModel({
     externalChannelUrl: req.body.externalChannelUrl,
     videoChannelId: req.body.videoChannelId,
-    state: VideoChannelSyncState.SYNCED
+    state: VideoChannelSyncState.WAITING_FIRST_RUN
   })
   await sequelizeTypescript.transaction(async t => {
     await syncCreated.save({ transaction: t })

server/initializers/constants.ts

@@ -211,10 +211,6 @@ const REPEAT_JOBS: { [ id in JobType ]?: EveryRepeatOptions | CronRepeatOptions
   },
   'activitypub-cleaner': {
     cron: '30 5 * * ' + randomInt(0, 7) // 1 time per week (random day) at 5:30 AM
-  },
-  'video-channel-sync-latest': {
-    // FIXME is this value realistic for every server? Should we allow the admin to configure it or is this fine as it is?
-    cron: '*/15 * * * *' // 1 time per 15 minutes
   }
 }
 const JOB_PRIORITY = {
@@ -878,7 +874,6 @@ if (process.env.PRODUCTION_CONSTANTS !== 'true') {
     REPEAT_JOBS['videos-views-stats'] = { every: 5000 }
 
     REPEAT_JOBS['activitypub-cleaner'] = { every: 5000 }
-    REPEAT_JOBS['video-channel-sync-latest'] = { every: 5 * 60 * 1000 }
     AP_CLEANER.PERIOD = 5000
 
     REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1

server/lib/job-queue/handlers/video-channels-sync.ts

@@ -260,5 +260,6 @@ async function addYoutubeDLImport (parameters: {
     videoImportId: videoImport.id,
     fileExt
   }
-  return JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
+  const job = await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
+  return job.finished()
 }

server/lib/job-queue/job-queue.ts

@@ -315,10 +315,6 @@ class JobQueue {
       repeat: REPEAT_JOBS['videos-views-stats']
     }).catch(err => logger.error('Cannot add repeatable job.', { err }))
 
-    this.queues['video-channel-sync-latest'].add({}, {
-      repeat: REPEAT_JOBS['video-channel-sync-latest']
-    }).catch(err => logger.error('Cannot add repeatable job.', { err }))
-
     if (CONFIG.FEDERATION.VIDEOS.CLEANUP_REMOTE_INTERACTIONS) {
       this.queues['activitypub-cleaner'].add({}, {
         repeat: REPEAT_JOBS['activitypub-cleaner']

server/tests/api/videos/video-channel-syncs.ts

@@ -6,7 +6,8 @@ import {
   createSingleServer,
   PeerTubeServer,
   setAccessTokensToServers,
-  setDefaultVideoChannel
+  setDefaultVideoChannel,
+  waitJobs
 } from '@shared/server-commands'
 import { expect } from 'chai'
 import 'mocha'
@@ -21,9 +22,11 @@ describe('Test channel synchronizations', function () {
       const userInfo = {
         accessToken: '',
         username: 'user1',
+        get channelName () {
+          return this.username + "_channel"
+        },
         channelId: -1,
-        syncId: -1,
-        syncs: []
+        syncId: -1
       }
 
       before(async function () {
@@ -99,10 +102,6 @@ describe('Test channel synchronizations', function () {
           `)
 
         // when
-        await command.syncChannel({
-          channelSyncId,
-          expectedStatus: HttpStatusCode.NO_CONTENT_204
-        })
         await server.debug.sendCommand({
           body: {
             command: 'process-video-channel-sync-latest'
@@ -113,6 +112,7 @@ describe('Test channel synchronizations', function () {
         // then
         {
           const res = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
+          await waitJobs(server, true)
           expect(res.total).to.equal(2)
           expect(res.data[0].name).to.equal('test')
         }
@@ -139,7 +139,7 @@ describe('Test channel synchronizations', function () {
           token: userInfo.accessToken,
           expectedStatus: HttpStatusCode.OK_200
         })
-        userInfo.syncs.push(id)
+        userInfo.syncId = id
       })
 
       it('Should list user\'s channel synchronizations', async function () {
@@ -161,17 +161,38 @@ describe('Test channel synchronizations', function () {
 
         const resForUser = await command.listByAccount({ accountName: userInfo.username })
         expect(resForUser.total).to.equal(1)
-        expect(resForUser.data[0]).to.contain({
-          externalChannelUrl: FIXTURE_URLS.youtubeChannel + "?baz=qux"
+        expect(resForUser.data[0]).to.deep.contain({
+          externalChannelUrl: FIXTURE_URLS.youtubeChannel + "?baz=qux",
+          state: {
+            id: VideoChannelSyncState.WAITING_FIRST_RUN,
+            label: 'Waiting first run'
+          }
+        })
+      })
+
+      it('Should not import a channel if not asked', async function () {
+        await waitJobs(server, true)
+        const resForUser = await command.listByAccount({ accountName: userInfo.username })
+        expect(resForUser.data[0].state).to.contain({
+          id: VideoChannelSyncState.WAITING_FIRST_RUN,
+          label: 'Waiting first run'
         })
       })
 
-      it('Should import a whole channel', async function () {
-        // TODO
+      it('Should import a whole channel (this test takes a while)', async function () {
+        this.timeout(240_000)
+        await command.syncChannel({
+          channelSyncId: userInfo.syncId
+        })
+        await waitJobs(server, true)
+        const resForUser = await server.videos.listByChannel({
+          handle: userInfo.channelName
+        })
+        expect(resForUser.total).to.equal(2)
       })
 
       it('Should remove user\'s channel synchronizations', async function () {
-        await command.delete({ channelSyncId: userInfo.syncs[0] })
+        await command.delete({ channelSyncId: userInfo.syncId })
         const resForUser = await command.listByAccount({ accountName: userInfo.username })
         expect(resForUser.total).to.equal(0)
       })

shared/server-commands/server/jobs.ts

@@ -17,7 +17,7 @@ async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer, skipDela
   const states: JobState[] = [ 'waiting', 'active' ]
   if (!skipDelayed) states.push('delayed')
 
-  const repeatableJobs: JobType[] = [ 'videos-views-stats', 'activitypub-cleaner', 'video-channel-sync-latest' ]
+  const repeatableJobs: JobType[] = [ 'videos-views-stats', 'activitypub-cleaner' ]
   let pendingRequests: boolean
 
   function tasksBuilder () {

shared/server-commands/videos/channel-syncs-command.ts

@@ -58,7 +58,6 @@ export class ChannelSyncsCommand extends AbstractCommand {
 
     return this.postBodyRequest({
       ...options,
-
       path,
       implicitToken: true,
       defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204