Merge branch 'upstream/process-ingest-data2' into release52

# Conflicts:
#	packages/blueprints-integration/src/api/studio.ts

meteor/__mocks__/defaultCollectionObjects.ts

@@ -134,7 +134,6 @@ export function defaultSegment(_id: SegmentId, rundownId: RundownId): DBSegment
 		externalId: unprotectString(_id),
 		rundownId: rundownId,
 		name: 'Default Segment',
-		externalModified: 1,
 	}
 }
 

meteor/__mocks__/helpers/database.ts

@@ -14,13 +14,11 @@ import {
 	SourceLayerType,
 	StudioBlueprintManifest,
 	BlueprintManifestType,
-	IngestRundown,
 	BlueprintManifestBase,
 	ShowStyleBlueprintManifest,
 	IShowStyleContext,
 	BlueprintResultRundown,
 	BlueprintResultSegment,
-	IngestSegment,
 	IBlueprintAdLibPiece,
 	IBlueprintRundown,
 	IBlueprintSegment,
@@ -32,6 +30,8 @@ import {
 	StatusCode,
 	IBlueprintPieceType,
 	IBlueprintActionManifest,
+	SofieIngestSegment,
+	SofieIngestRundown,
 } from '@sofie-automation/blueprints-integration'
 import { DBShowStyleBase } from '@sofie-automation/corelib/dist/dataModel/ShowStyleBase'
 import { DBShowStyleVariant } from '@sofie-automation/corelib/dist/dataModel/ShowStyleVariant'
@@ -429,7 +429,10 @@ export async function setupMockShowStyleBlueprint(
 				getShowStyleVariantId: (): string | null => {
 					return SHOW_STYLE_VARIANT_ID
 				},
-				getRundown: (_context: IShowStyleContext, ingestRundown: IngestRundown): BlueprintResultRundown => {
+				getRundown: (
+					_context: IShowStyleContext,
+					ingestRundown: SofieIngestRundown<any, any, any>
+				): BlueprintResultRundown => {
 					const rundown: IBlueprintRundown = {
 						externalId: ingestRundown.externalId,
 						name: ingestRundown.name,
@@ -452,7 +455,10 @@ export async function setupMockShowStyleBlueprint(
 						baseline: { timelineObjects: [] },
 					}
 				},
-				getSegment: (_context: unknown, ingestSegment: IngestSegment): BlueprintResultSegment => {
+				getSegment: (
+					_context: unknown,
+					ingestSegment: SofieIngestSegment<any, any>
+				): BlueprintResultSegment => {
 					const segment: IBlueprintSegment = {
 						name: ingestSegment.name ? ingestSegment.name : ingestSegment.externalId,
 						privateData: ingestSegment.payload,
@@ -654,7 +660,6 @@ export async function setupDefaultRundown(
 		externalId: 'MOCK_SEGMENT_0',
 		rundownId: rundown._id,
 		name: 'Segment 0',
-		externalModified: 1,
 	}
 	await Segments.mutableCollection.insertAsync(segment0)
 	/* tslint:disable:ter-indent*/
@@ -764,7 +769,6 @@ export async function setupDefaultRundown(
 		externalId: 'MOCK_SEGMENT_2',
 		rundownId: rundown._id,
 		name: 'Segment 1',
-		externalModified: 1,
 	}
 	await Segments.mutableCollection.insertAsync(segment1)
 
@@ -807,7 +811,6 @@ export async function setupDefaultRundown(
 		externalId: 'MOCK_SEGMENT_2',
 		rundownId: rundown._id,
 		name: 'Segment 2',
-		externalModified: 1,
 	}
 	await Segments.mutableCollection.insertAsync(segment2)
 

meteor/server/__tests__/_testEnvironment.test.ts

@@ -11,7 +11,7 @@ import {
 	Evaluations,
 	ExpectedMediaItems,
 	ExternalMessageQueue,
-	IngestDataCache,
+	NrcsIngestDataCache,
 	MediaObjects,
 	MediaWorkFlows,
 	MediaWorkFlowSteps,
@@ -74,7 +74,7 @@ describe('Basic test of test environment', () => {
 		// @ts-ignore
 		expect(ExternalMessageQueue._isMock).toBeTruthy()
 		// @ts-ignore
-		expect(IngestDataCache._isMock).toBeTruthy()
+		expect(NrcsIngestDataCache._isMock).toBeTruthy()
 		// @ts-ignore
 		expect(MediaObjects._isMock).toBeTruthy()
 		// @ts-ignore

meteor/server/__tests__/cronjobs.test.ts

@@ -19,7 +19,8 @@ import { PieceInstance } from '@sofie-automation/corelib/dist/dataModel/PieceIns
 import { Meteor } from 'meteor/meteor'
 import { EmptyPieceTimelineObjectsBlob } from '@sofie-automation/corelib/dist/dataModel/Piece'
 import {
-	IngestDataCacheObjId,
+	NrcsIngestDataCacheObjId,
+	SofieIngestDataCacheObjId,
 	PartId,
 	PeripheralDeviceId,
 	RundownId,
@@ -42,7 +43,7 @@ import '../cronjobs'
 import '../api/peripheralDevice'
 import {
 	CoreSystem,
-	IngestDataCache,
+	NrcsIngestDataCache,
 	PartInstances,
 	Parts,
 	PeripheralDeviceCommands,
@@ -51,8 +52,9 @@ import {
 	Snapshots,
 	UserActionsLog,
 	Segments,
+	SofieIngestDataCache,
 } from '../collections'
-import { IngestCacheType } from '@sofie-automation/corelib/dist/dataModel/IngestDataCache'
+import { NrcsIngestCacheType } from '@sofie-automation/corelib/dist/dataModel/NrcsIngestDataCache'
 import { JSONBlobStringify } from '@sofie-automation/shared-lib/dist/lib/JSONBlob'
 import {
 	DefaultEnvironment,
@@ -61,6 +63,7 @@ import {
 } from '../../__mocks__/helpers/database'
 import { DBSegment } from '@sofie-automation/corelib/dist/dataModel/Segment'
 import { Settings } from '../Settings'
+import { SofieIngestCacheType } from '@sofie-automation/corelib/dist/dataModel/SofieIngestDataCache'
 
 describe('cronjobs', () => {
 	let env: DefaultEnvironment
@@ -170,45 +173,95 @@ describe('cronjobs', () => {
 			await PeripheralDevices.removeAsync({})
 		})
 
-		testInFiber('Remove IngestDataCache objects that are not connected to any Rundown', async () => {
-			// Set up a mock rundown, a detached IngestDataCache object and an object attached to the mock rundown
-			// Detached IngestDataCache object 0
-			const dataCache0Id = protectString<IngestDataCacheObjId>(getRandomString())
-			await IngestDataCache.mutableCollection.insertAsync({
+		testInFiber('Remove NrcsIngestDataCache objects that are not connected to any Rundown', async () => {
+			// Set up a mock rundown, a detached NrcsIngestDataCache object and an object attached to the mock rundown
+			// Detached NrcsIngestDataCache object 0
+			const dataCache0Id = protectString<NrcsIngestDataCacheObjId>(getRandomString())
+			await NrcsIngestDataCache.mutableCollection.insertAsync({
 				_id: dataCache0Id,
 				data: {
 					externalId: '',
 					name: '',
 					segments: [],
 					type: '',
+					rundownSource: {} as any,
+					payload: undefined,
 				},
 				modified: new Date(2000, 0, 1, 0, 0, 0).getTime(),
 				// this one is attached to rundown0
 				rundownId: getRandomId(),
-				type: IngestCacheType.RUNDOWN,
+				type: NrcsIngestCacheType.RUNDOWN,
 			})
-			// Attached IngestDataCache object 1
-			const dataCache1Id = protectString<IngestDataCacheObjId>(getRandomString())
-			await IngestDataCache.mutableCollection.insertAsync({
+			// Attached NrcsIngestDataCache object 1
+			const dataCache1Id = protectString<NrcsIngestDataCacheObjId>(getRandomString())
+			await NrcsIngestDataCache.mutableCollection.insertAsync({
 				_id: dataCache1Id,
 				data: {
 					externalId: '',
 					name: '',
 					segments: [],
 					type: '',
+					rundownSource: {} as any,
+					payload: undefined,
 				},
 				modified: new Date(2000, 0, 1, 0, 0, 0).getTime(),
 				// just some random ID
 				rundownId: rundownId,
-				type: IngestCacheType.RUNDOWN,
+				type: NrcsIngestCacheType.RUNDOWN,
 			})
 
 			await runCronjobs()
 
-			expect(await IngestDataCache.findOneAsync(dataCache1Id)).toMatchObject({
+			expect(await NrcsIngestDataCache.findOneAsync(dataCache1Id)).toMatchObject({
 				_id: dataCache1Id,
 			})
-			expect(await IngestDataCache.findOneAsync(dataCache0Id)).toBeUndefined()
+			expect(await NrcsIngestDataCache.findOneAsync(dataCache0Id)).toBeUndefined()
+		})
+		testInFiber('Remove SofieIngestDataCache objects that are not connected to any Rundown', async () => {
+			// Set up a mock rundown, a detached SofieIngestDataCache object and an object attached to the mock rundown
+			// Detached SofieIngestDataCache object 0
+			const dataCache0Id = protectString<SofieIngestDataCacheObjId>(getRandomString())
+			await SofieIngestDataCache.mutableCollection.insertAsync({
+				_id: dataCache0Id,
+				data: {
+					externalId: '',
+					name: '',
+					segments: [],
+					type: '',
+					rundownSource: {} as any,
+					userEditStates: {},
+					payload: undefined,
+				},
+				modified: new Date(2000, 0, 1, 0, 0, 0).getTime(),
+				// this one is attached to rundown0
+				rundownId: getRandomId(),
+				type: SofieIngestCacheType.RUNDOWN,
+			})
+			// Attached SofieIngestDataCache object 1
+			const dataCache1Id = protectString<SofieIngestDataCacheObjId>(getRandomString())
+			await SofieIngestDataCache.mutableCollection.insertAsync({
+				_id: dataCache1Id,
+				data: {
+					externalId: '',
+					name: '',
+					segments: [],
+					type: '',
+					rundownSource: {} as any,
+					userEditStates: {},
+					payload: undefined,
+				},
+				modified: new Date(2000, 0, 1, 0, 0, 0).getTime(),
+				// just some random ID
+				rundownId: rundownId,
+				type: SofieIngestCacheType.RUNDOWN,
+			})
+
+			await runCronjobs()
+
+			expect(await SofieIngestDataCache.findOneAsync(dataCache1Id)).toMatchObject({
+				_id: dataCache1Id,
+			})
+			expect(await SofieIngestDataCache.findOneAsync(dataCache0Id)).toBeUndefined()
 		})
 		testInFiber('Removes old PartInstances and PieceInstances', async () => {
 			// nightlyCronjobInner()
@@ -217,7 +270,6 @@ describe('cronjobs', () => {
 				_id: getRandomId<SegmentId>(),
 				_rank: 0,
 				externalId: '',
-				externalModified: 0,
 				rundownId,
 				name: 'mock segment',
 			}

meteor/server/api/__tests__/cleanup.test.ts

@@ -32,7 +32,7 @@ import {
 	ExpectedPackageWorkStatuses,
 	ExpectedPlayoutItems,
 	ExternalMessageQueue,
-	IngestDataCache,
+	NrcsIngestDataCache,
 	PackageContainerPackageStatuses,
 	PackageInfos,
 	PeripheralDeviceCommands,
@@ -45,6 +45,7 @@ import {
 	TranslationsBundles,
 	PackageContainerStatuses,
 	TimelineDatastore,
+	SofieIngestDataCache,
 } from '../../collections'
 import { Collections } from '../../collections/lib'
 import { generateTranslationBundleOriginId } from '../translationsBundles'
@@ -300,7 +301,14 @@ async function setDefaultDatatoDB(env: DefaultEnvironment, now: number) {
 		tryCount: 0,
 		type: '' as any,
 	})
-	await IngestDataCache.mutableCollection.insertAsync({
+	await NrcsIngestDataCache.mutableCollection.insertAsync({
+		_id: getRandomId(),
+		data: {} as any,
+		modified: 0,
+		rundownId,
+		type: '' as any,
+	})
+	await SofieIngestDataCache.mutableCollection.insertAsync({
 		_id: getRandomId(),
 		data: {} as any,
 		modified: 0,

meteor/server/api/__tests__/peripheralDevice.test.ts

@@ -120,7 +120,6 @@ describe('test peripheralDevice general API methods', () => {
 			_rank: 0,
 			rundownId: rundownID,
 			name: 'Fire',
-			externalModified: 1,
 		})
 		await Parts.mutableCollection.insertAsync({
 			_id: protectString('part000'),
@@ -164,15 +163,13 @@ describe('test peripheralDevice general API methods', () => {
 			externalId: 'segment01',
 			rundownId: rundownID,
 			name: 'Water',
-			externalModified: 1,
 		})
 		await Segments.mutableCollection.insertAsync({
 			_id: protectString('segment2'),
 			_rank: 2,
 			externalId: 'segment02',
 			rundownId: rundownID,
 			name: 'Earth',
-			externalModified: 1,
 		})
 	})
 	beforeEach(async () => {

meteor/server/api/cleanup.ts

@@ -37,7 +37,7 @@ import {
 	ExpectedPackageWorkStatuses,
 	ExpectedPlayoutItems,
 	ExternalMessageQueue,
-	IngestDataCache,
+	NrcsIngestDataCache,
 	MediaObjects,
 	MediaWorkFlows,
 	MediaWorkFlowSteps,
@@ -69,6 +69,7 @@ import {
 	UserActionsLog,
 	Workers,
 	WorkerThreadStatuses,
+	SofieIngestDataCache,
 } from '../collections'
 import { AsyncOnlyMongoCollection, AsyncOnlyReadOnlyMongoCollection } from '../collections/collection'
 import { getCollectionKey } from '../collections/lib'
@@ -276,7 +277,8 @@ export async function cleanupOldDataInner(actuallyCleanup = false): Promise<Coll
 		}
 		await ownedByRundownId(AdLibActions)
 		await ownedByRundownId(AdLibPieces)
-		await ownedByRundownId(IngestDataCache)
+		await ownedByRundownId(SofieIngestDataCache)
+		await ownedByRundownId(NrcsIngestDataCache)
 		;(await ownedByRundownId(Parts)).forEach((id) => removedParts.add(id))
 		await ownedByRundownId(RundownBaselineAdLibActions)
 		await ownedByRundownId(RundownBaselineAdLibPieces)

meteor/server/api/client.ts

@@ -422,6 +422,7 @@ export namespace ServerClientAPI {
 
 class ServerClientAPIClass extends MethodContextAPI implements NewClientAPI {
 	async clientLogger(type: string, ...args: string[]): Promise<void> {
+		triggerWriteAccessBecauseNoCheckNecessary()
 		const loggerFunction: LeveledLogMethodFixed = (logger as any)[type] || logger.log
 
 		loggerFunction(args.join(', '))