backport sync fixes to stable31

cypress/component/helpers/yjs.cy.js

@@ -4,7 +4,7 @@
  */
 
 import * as Y from 'yjs'
-import { getDocumentState, documentStateToStep, applyStep } from '../../../src/helpers/yjs.js'
+import { getDocumentState, documentStateToStep, applyStep } from '../../../src/helpers/yjs.ts'
 
 describe('Yjs base64 wrapped with our helpers', function() {
 	it('applies step generated from document state', function() {

cypress/e2e/api/SyncServiceProvider.spec.js

@@ -5,7 +5,7 @@
 
 import { randUser } from '../../utils/index.js'
 import SessionApi from '../../../src/services/SessionApi.js'
-import { SyncService } from '../../../src/services/SyncService.js'
+import { SyncService } from '../../../src/services/SyncService.ts'
 import createSyncServiceProvider from '../../../src/services/SyncServiceProvider.js'
 import { Doc } from 'yjs'
 
@@ -46,7 +46,7 @@ describe('Sync service provider', function() {
 			getDocumentState: () => null,
 			api,
 		})
-		syncService.on('opened', () => syncService.startSync())
+		syncService.bus.on('opened', () => syncService.startSync())
 		return createSyncServiceProvider({
 			ydoc,
 			syncService,

lib/Controller/PublicSessionController.php

@@ -78,8 +78,8 @@ public function close(int $documentId, int $sessionId, string $sessionToken): Da
 	#[PublicPage]
 	#[RequireDocumentBaseVersionEtag]
 	#[RequireDocumentSession]
-	public function push(int $documentId, int $sessionId, string $sessionToken, int $version, array $steps, string $awareness, string $token): DataResponse {
-		return $this->apiService->push($this->getSession(), $this->getDocument(), $version, $steps, $awareness, $token);
+	public function push(int $documentId, int $sessionId, string $sessionToken, int $version, array $steps, string $awareness, string $token, ?int $recoveryAttempt = null): DataResponse {
+		return $this->apiService->push($this->getSession(), $this->getDocument(), $version, $steps, $awareness, $recoveryAttempt, $token);
 	}
 
 	#[NoAdminRequired]

lib/Controller/SessionController.php

@@ -56,10 +56,10 @@ public function close(int $documentId, int $sessionId, string $sessionToken): Da
 	#[PublicPage]
 	#[RequireDocumentBaseVersionEtag]
 	#[RequireDocumentSession]
-	public function push(int $version, array $steps, string $awareness): DataResponse {
+	public function push(int $version, array $steps, string $awareness, ?int $recoveryAttempt = null): DataResponse {
 		try {
 			$this->loginSessionUser();
-			return $this->apiService->push($this->getSession(), $this->getDocument(), $version, $steps, $awareness);
+			return $this->apiService->push($this->getSession(), $this->getDocument(), $version, $steps, $awareness, $recoveryAttempt);
 		} finally {
 			$this->restoreSessionUser();
 		}

lib/Service/ApiService.php

@@ -174,15 +174,15 @@ public function close(int $documentId, int $sessionId, string $sessionToken): Da
 	/**
 	 * @throws NotFoundException
 	 */
-	public function push(Session $session, Document $document, int $version, array $steps, string $awareness, ?string $token = null): DataResponse {
+	public function push(Session $session, Document $document, int $version, array $steps, string $awareness, ?int $recoveryAttempt, ?string $token = null): DataResponse {
 		try {
 			$session = $this->sessionService->updateSessionAwareness($session, $awareness);
 		} catch (DoesNotExistException $e) {
 			// Session was removed in the meantime. #3875
 			return new DataResponse(['error' => $this->l10n->t('Editing session has expired. Please reload the page.')], Http::STATUS_PRECONDITION_FAILED);
 		}
 		try {
-			$result = $this->documentService->addStep($document, $session, $steps, $version, $token);
+			$result = $this->documentService->addStep($document, $session, $steps, $version, $recoveryAttempt, $token);
 			$this->addToPushQueue($document, [$awareness, ...array_values($steps)]);
 		} catch (InvalidArgumentException $e) {
 			return new DataResponse(['error' => $e->getMessage()], Http::STATUS_UNPROCESSABLE_ENTITY);

lib/Service/DocumentService.php

@@ -205,7 +205,7 @@ public function writeDocumentState(int $documentId, string $content): void {
 	 * @throws NotPermittedException
 	 * @throws DoesNotExistException
 	 */
-	public function addStep(Document $document, Session $session, array $steps, int $version, ?string $shareToken): array {
+	public function addStep(Document $document, Session $session, array $steps, int $version, ?int $recoveryAttempt, ?string $shareToken): array {
 		$documentId = $session->getDocumentId();
 		$readOnly = $this->isReadOnlyCached($session, $shareToken);
 		$stepsToInsert = [];
@@ -234,6 +234,11 @@ public function addStep(Document $document, Session $session, array $steps, int
 		// By default, send all steps the user has not received yet.
 		$getStepsSinceVersion = $version;
 		if ($stepsIncludeQuery) {
+			if ($recoveryAttempt === 1) {
+				$this->logger->error('Recovery attempt #' . $recoveryAttempt . ' from ' . $session->getId() . ' for ' . $documentId);
+			} elseif ($recoveryAttempt > 1) {
+				$this->logger->debug('Recovery attempt #' . $recoveryAttempt . ' from ' . $session->getId() . ' for ' . $documentId);
+			}
 			$this->logger->debug('Loading document state for ' . $documentId);
 			try {
 				$stateFile = $this->getStateFile($documentId);

src/components/Editor.vue

@@ -96,8 +96,8 @@ import {
 import ReadonlyBar from './Menu/ReadonlyBar.vue'
 
 import { logger } from '../helpers/logger.js'
-import { getDocumentState } from '../helpers/yjs.js'
-import { SyncService, ERROR_TYPE, IDLE_TIMEOUT } from './../services/SyncService.js'
+import { applyDocumentState, getDocumentState } from '../helpers/yjs.ts'
+import { SyncService, ERROR_TYPE, IDLE_TIMEOUT } from './../services/SyncService.ts'
 import SessionApi from '../services/SessionApi.js'
 import createSyncServiceProvider from './../services/SyncServiceProvider.js'
 import AttachmentResolver from './../services/AttachmentResolver.js'
@@ -462,27 +462,27 @@ export default {
 		},
 
 		listenSyncServiceEvents() {
-			this.$syncService
-				.on('opened', this.onOpened)
-				.on('change', this.onChange)
-				.on('loaded', this.onLoaded)
-				.on('sync', this.onSync)
-				.on('error', this.onError)
-				.on('stateChange', this.onStateChange)
-				.on('idle', this.onIdle)
-				.on('save', this.onSave)
+			const bus = this.$syncService.bus
+			bus.on('opened', this.onOpened)
+			bus.on('change', this.onChange)
+			bus.on('loaded', this.onLoaded)
+			bus.on('sync', this.onSync)
+			bus.on('error', this.onError)
+			bus.on('stateChange', this.onStateChange)
+			bus.on('idle', this.onIdle)
+			bus.on('save', this.onSave)
 		},
 
 		unlistenSyncServiceEvents() {
-			this.$syncService
-				.off('opened', this.onOpened)
-				.off('change', this.onChange)
-				.off('loaded', this.onLoaded)
-				.off('sync', this.onSync)
-				.off('error', this.onError)
-				.off('stateChange', this.onStateChange)
-				.off('idle', this.onIdle)
-				.off('save', this.onSave)
+			const bus = this.$syncService.bus
+			bus.off('opened', this.onOpened)
+			bus.off('change', this.onChange)
+			bus.off('loaded', this.onLoaded)
+			bus.off('sync', this.onSync)
+			bus.off('error', this.onError)
+			bus.off('stateChange', this.onStateChange)
+			bus.off('idle', this.onIdle)
+			bus.off('save', this.onSave)
 		},
 
 		reconnect() {
@@ -556,7 +556,9 @@ export default {
 		},
 
 		onLoaded({ document, documentSource, documentState }) {
-			if (!documentState) {
+			if (documentState) {
+				applyDocumentState(this.$ydoc, documentState, this.$providers[0])
+			} else {
 				this.setInitialYjsState(documentSource, { isRichEditor: this.isRichEditor })
 			}
 

src/components/Editor/DocumentStatus/SyncStatus.vue

@@ -14,7 +14,7 @@
 
 <script>
 
-import { ERROR_TYPE } from '../../../services/SyncService.js'
+import { ERROR_TYPE } from '../../../services/SyncService.ts'
 import NcNoteCard from '@nextcloud/vue/components/NcNoteCard'
 
 export default {

src/components/Editor/SessionList.vue

@@ -55,7 +55,7 @@ import NcCheckboxRadioSwitch from '@nextcloud/vue/components/NcCheckboxRadioSwit
 import NcPopover from '@nextcloud/vue/components/NcPopover'
 import AccountMultipleIcon from 'vue-material-design-icons/AccountMultiple.vue'
 import AvatarWrapper from './AvatarWrapper.vue'
-import { COLLABORATOR_DISCONNECT_TIME, COLLABORATOR_IDLE_TIME } from '../../services/SyncService.js'
+import { COLLABORATOR_DISCONNECT_TIME, COLLABORATOR_IDLE_TIME } from '../../services/SyncService.ts'
 import { loadState } from '@nextcloud/initial-state'
 import axios from '@nextcloud/axios'
 import { generateUrl } from '@nextcloud/router'

src/components/Editor/Status.vue

@@ -27,7 +27,7 @@
 
 <script>
 
-import { ERROR_TYPE } from '../../services/SyncService.js'
+import { ERROR_TYPE } from '../../services/SyncService.ts'
 import moment from '@nextcloud/moment'
 import NcButton from '@nextcloud/vue/components/NcButton'
 import NcSavingIndicatorIcon from '@nextcloud/vue/components/NcSavingIndicatorIcon'

src/helpers/base64.ts

@@ -17,7 +17,7 @@ import { toBase64, fromBase64 } from 'lib0/buffer'
  *
  * @param {ArrayBuffer} data - binary data to encode
  */
-export function encodeArrayBuffer(data: ArrayBuffer): string {
+export function encodeArrayBuffer(data: Uint8Array<ArrayBufferLike>): string {
 	const view = new Uint8Array(data)
 	return toBase64(view)
 }

src/helpers/files.js

@@ -166,7 +166,7 @@ let FilesHeaderRichWorkspaceView
 let FilesHeaderRichWorkspaceInstance
 let latestFolder
 
-function enabled(_, view) {
+const enabled = (_, view) => {
 	return ['files', 'favorites', 'public-share'].includes(view.id)
 }
 

src/helpers/steps.ts

@@ -0,0 +1,24 @@
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+import type { Session, Step } from '../services/SyncService'
+import { COLLABORATOR_DISCONNECT_TIME } from '../services/SyncService'
+
+/**
+ * Get the recent awareness messages as steps
+ * @param sessions to process.
+ */
+export function awarenessSteps(sessions: Session[]): Step[] {
+	const lastContactThreshold
+		= Math.floor(Date.now() / 1000) - COLLABORATOR_DISCONNECT_TIME
+	return sessions
+		.filter((s) => s.lastContact > lastContactThreshold)
+		.filter((s) => Boolean(s.lastAwarenessMessage))
+		.map((s) => ({
+			data: [s.lastAwarenessMessage],
+			sessionId: s.id,
+			version: 0,
+		}))
+}

src/helpers/yjs.js → src/helpers/yjs.ts

@@ -3,32 +3,36 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
-import { encodeArrayBuffer, decodeArrayBuffer } from '../helpers/base64.ts'
-import * as Y from 'yjs'
 import * as decoding from 'lib0/decoding.js'
 import * as encoding from 'lib0/encoding.js'
 import * as syncProtocol from 'y-protocols/sync'
+import * as Y from 'yjs'
+import type { Step } from '../services/SyncService'
 import { messageSync } from '../services/y-websocket.js'
+import { decodeArrayBuffer, encodeArrayBuffer } from './base64'
 
 /**
  * Get Document state encode as base64.
  *
  * Used to store yjs state on the server.
- * @param {Y.Doc} ydoc - encode state of this doc
- * @return {string}
+ * @param ydoc - encode state of this doc
  */
-export function getDocumentState(ydoc) {
+export function getDocumentState(ydoc: Y.Doc): string {
 	const update = Y.encodeStateAsUpdate(ydoc)
 	return encodeArrayBuffer(update)
 }
 
 /**
  *
- * @param {Y.Doc} ydoc - apply state to this doc
- * @param {string} documentState - base64 encoded doc state
- * @param {object} origin - initiator object e.g. WebsocketProvider
+ * @param ydoc - apply state to this doc
+ * @param documentState - base64 encoded doc state
+ * @param origin - initiator object e.g. WebsocketProvider
  */
-export function applyDocumentState(ydoc, documentState, origin) {
+export function applyDocumentState(
+	ydoc: Y.Doc,
+	documentState: string,
+	origin: object,
+) {
 	const update = decodeArrayBuffer(documentState)
 	Y.applyUpdate(ydoc, update, origin)
 }
@@ -38,23 +42,22 @@ export function applyDocumentState(ydoc, documentState, origin) {
  * i.e. create a sync protocol update message from it
  * and encode it and wrap it in a step data structure.
  *
- * @param {string} documentState - base64 encoded doc state
- * @return {string} base64 encoded yjs sync protocol update message
+ * @param documentState - base64 encoded doc state
+ * @return base64 encoded yjs sync protocol update message and version
  */
-export function documentStateToStep(documentState) {
+export function documentStateToStep(documentState: string): Step {
 	const message = documentStateToUpdateMessage(documentState)
-	return { step: encodeArrayBuffer(message) }
+	return { data: [encodeArrayBuffer(message)], sessionId: 0, version: -1 }
 }
 
 /**
- * Create an update message from a document state
+ * Create a message from a document state
  * i.e. decode the base64 encoded yjs update
  * and create a sync protocol update message from it
  *
- * @param {string} documentState - base64 encoded doc state
- * @return {Uint8Array}
+ * @param documentState - base64 encoded doc state
  */
-function documentStateToUpdateMessage(documentState) {
+function documentStateToUpdateMessage(documentState: string): Uint8Array {
 	const update = decodeArrayBuffer(documentState)
 	const encoder = encoding.createEncoder()
 	encoding.writeVarUint(encoder, messageSync)
@@ -66,34 +69,32 @@ function documentStateToUpdateMessage(documentState) {
  * Apply a step to the ydoc.
  *
  * Only used in tests right now.
- * @param {Y.Doc} ydoc - encode state of this doc
- * @param {string} step - base64 encoded yjs sync update message
- * @param {object} origin - initiator object e.g. WebsocketProvider
+ * @param ydoc - encode state of this doc
+ * @param step - step data
+ * @param step.data - array of base64 encoded yjs sync update messages
+ * @param origin - initiator object e.g. WebsocketProvider
  */
-export function applyStep(ydoc, step, origin = 'origin') {
-	const updateMessage = decodeArrayBuffer(step.step)
-	const decoder = decoding.createDecoder(updateMessage)
-	const messageType = decoding.readVarUint(decoder)
-	if (messageType !== messageSync) {
-		console.error('y.js update message with invalid type', messageType)
-		return
+export function applyStep(ydoc: Y.Doc, step: Step, origin = 'origin') {
+	for (const encoded of step.data) {
+		const updateMessage = decodeArrayBuffer(encoded)
+		const decoder = decoding.createDecoder(updateMessage)
+		const messageType = decoding.readVarUint(decoder)
+		if (messageType !== messageSync) {
+			console.error('y.js update message with invalid type', messageType)
+			return
+		}
+		// There are no responses to updates - so this is a dummy.
+		const encoder = encoding.createEncoder()
+		syncProtocol.readSyncMessage(decoder, encoder, ydoc, origin)
 	}
-	// There are no responses to updates - so this is a dummy.
-	const encoder = encoding.createEncoder()
-	syncProtocol.readSyncMessage(
-		decoder,
-		encoder,
-		ydoc,
-		origin,
-	)
 }
 
 /**
  * Log y.js messages with their type and initiator call stack
  *
- * @param {string} step - Y.js message
+ * @param step - Y.js message
  */
-export function logStep(step) {
+export function logStep(step: Uint8Array<ArrayBufferLike>) {
 	// Create error for stack trace
 	const err = new Error()