diff --git a/lib/plugin-support/shared-workers.js b/lib/plugin-support/shared-workers.js
index 94030ced5..1a23fa230 100644
--- a/lib/plugin-support/shared-workers.js
+++ b/lib/plugin-support/shared-workers.js
@@ -51,8 +51,13 @@ function launchWorker(filename, initialData) {
 export async function observeWorkerProcess(fork, runStatus) {
 	let registrationCount = 0;
 	let signalDeregistered;
+	let launched;
 	const deregistered = new Promise(resolve => {
-		signalDeregistered = resolve;
+		signalDeregistered = () => {
+			// Only unref the worker once all test workers have been deregistered, otherwise the worker may exit before test workers are deregistered
+			launched?.worker.unref();
+			resolve();
+		};
 	});
 
 	fork.promise.finally(() => {
@@ -62,7 +67,7 @@ export async function observeWorkerProcess(fork, runStatus) {
 	});
 
 	fork.onConnectSharedWorker(async ({filename, initialData, port, signalError}) => {
-		const launched = launchWorker(filename, initialData);
+		launched = launchWorker(filename, initialData);
 
 		const handleWorkerMessage = async message => {
 			if (message.type === 'deregistered-test-worker' && message.id === fork.threadId) {
@@ -104,14 +109,7 @@ export async function observeWorkerProcess(fork, runStatus) {
 			});
 
 			launched.worker.on('message', handleWorkerMessage);
-		} catch {
-			return;
-		} finally {
-			// Attaching listeners has the side-effect of referencing the worker.
-			// Explicitly unreference it now so it does not prevent the main process
-			// from exiting.
-			launched.worker.unref();
-		}
+		} catch {}
 	});
 
 	return deregistered;
diff --git a/test/shared-workers/timeout/fixtures/package.json b/test/shared-workers/timeout/fixtures/package.json
new file mode 100644
index 000000000..3e7bec51a
--- /dev/null
+++ b/test/shared-workers/timeout/fixtures/package.json
@@ -0,0 +1,9 @@
+{
+	"type": "module",
+	"ava": {
+		"files": [
+			"*.js"
+		],
+		"timeout": "5s"
+	}
+}
diff --git a/test/shared-workers/timeout/fixtures/test.js b/test/shared-workers/timeout/fixtures/test.js
new file mode 100644
index 000000000..60f7d9e38
--- /dev/null
+++ b/test/shared-workers/timeout/fixtures/test.js
@@ -0,0 +1,14 @@
+import test from 'ava';
+import {registerSharedWorker} from 'ava/plugin';
+import delay from 'delay';
+
+registerSharedWorker({
+	filename: new URL('worker.mjs', import.meta.url),
+	supportedProtocols: ['ava-4'],
+});
+
+test('time out', async t => {
+	await delay(10_000);
+
+	t.pass();
+});
diff --git a/test/shared-workers/timeout/fixtures/worker.mjs b/test/shared-workers/timeout/fixtures/worker.mjs
new file mode 100644
index 000000000..fa2d5cabd
--- /dev/null
+++ b/test/shared-workers/timeout/fixtures/worker.mjs
@@ -0,0 +1,3 @@
+export default ({negotiateProtocol}) => {
+	negotiateProtocol(['ava-4']).ready();
+};
diff --git a/test/shared-workers/timeout/test.js b/test/shared-workers/timeout/test.js
new file mode 100644
index 000000000..a90d673ee
--- /dev/null
+++ b/test/shared-workers/timeout/test.js
@@ -0,0 +1,9 @@
+import test from '@ava/test';
+
+import {fixture} from '../../helpers/exec.js';
+
+test('times out', async t => {
+	const error = await t.throwsAsync(async () => fixture());
+
+	t.is(error.exitCode, 1);
+});