fix: resolve environment switching issues with stale server connections

Fixed three critical bugs that prevented proper environment switching:

1. Controller disposal race condition: Old controller was disposed before
   the new controller was ready, causing "DISPOSED" errors during cell
   execution. Fixed by deferring disposal until after new controller is
   fully registered.

2. Stale configuration caching: Configuration object wasn't refreshed after
   startServer(), so we connected with outdated serverInfo. Fixed by always
   calling getEnvironment() after startServer() to get current server info.

3. Environment manager early return: startServer() had an early return when
   config.serverInfo was set, preventing verification that the server was
   actually running. This caused connections to wrong/stale servers when
   switching TO a previously-used environment. Fixed by always calling
   serverStarter.startServer() (which is idempotent) to ensure current info.

Additional improvements:
- Made kernel spec installation idempotent and ensured it runs when reusing
  existing venvs
- Removed legacy auto-create fallback path that's no longer needed
- Added proper TypeScript non-null assertions after server info validation

Author: hc2p

src/kernels/deepnote/deepnoteToolkitInstaller.node.ts

@@ -95,7 +95,17 @@ export class DeepnoteToolkitInstaller implements IDeepnoteToolkitInstaller {
         // Check if venv already exists with toolkit installed
         const existingVenv = await this.getVenvInterpreterByPath(venvPath);
         if (existingVenv && (await this.isToolkitInstalled(existingVenv))) {
-            logger.info(`deepnote-toolkit venv already exists and is ready at ${venvPath.fsPath}`);
+            logger.info(`deepnote-toolkit venv already exists at ${venvPath.fsPath}`);
+
+            // Ensure kernel spec is installed (may have been deleted or never installed)
+            try {
+                await this.installKernelSpec(existingVenv, venvPath);
+            } catch (ex) {
+                logger.warn(`Failed to ensure kernel spec installed: ${ex}`);
+                // Don't fail - continue with existing venv
+            }
+
+            logger.info(`Venv ready at ${venvPath.fsPath}`);
             return existingVenv;
         }
 
@@ -288,30 +298,8 @@ export class DeepnoteToolkitInstaller implements IDeepnoteToolkitInstaller {
                 logger.info('deepnote-toolkit installed successfully in venv');
 
                 // Install kernel spec so the kernel uses this venv's Python
-                // Install into the venv itself (not --user) so the Deepnote server can discover it
-                logger.info('Installing kernel spec for venv...');
                 try {
-                    const kernelSpecName = this.getKernelSpecName(venvPath);
-                    const kernelDisplayName = this.getKernelDisplayName(venvPath);
-
-                    // Reuse the process service with system environment
-                    await venvProcessService.exec(
-                        venvInterpreter.uri.fsPath,
-                        [
-                            '-m',
-                            'ipykernel',
-                            'install',
-                            '--prefix',
-                            venvPath.fsPath,
-                            '--name',
-                            kernelSpecName,
-                            '--display-name',
-                            kernelDisplayName
-                        ],
-                        { throwOnStdErr: false }
-                    );
-                    const kernelSpecPath = Uri.joinPath(venvPath, 'share', 'jupyter', 'kernels', kernelSpecName);
-                    logger.info(`Kernel spec installed successfully to ${kernelSpecPath.fsPath}`);
+                    await this.installKernelSpec(venvInterpreter, venvPath);
                 } catch (ex) {
                     logger.warn(`Failed to install kernel spec: ${ex}`);
                     // Don't fail the entire installation if kernel spec creation fails
@@ -364,6 +352,43 @@ export class DeepnoteToolkitInstaller implements IDeepnoteToolkitInstaller {
         return `Deepnote (${venvDirName})`;
     }
 
+    /**
+     * Install ipykernel kernel spec for a venv.
+     * This is idempotent - safe to call multiple times.
+     */
+    private async installKernelSpec(venvInterpreter: PythonEnvironment, venvPath: Uri): Promise<void> {
+        const kernelSpecName = this.getKernelSpecName(venvPath);
+        const kernelSpecPath = Uri.joinPath(venvPath, 'share', 'jupyter', 'kernels', kernelSpecName);
+
+        // Check if kernel spec already exists
+        if (await this.fs.exists(kernelSpecPath)) {
+            logger.info(`Kernel spec already exists at ${kernelSpecPath.fsPath}`);
+            return;
+        }
+
+        logger.info(`Installing kernel spec '${kernelSpecName}' for venv at ${venvPath.fsPath}...`);
+        const kernelDisplayName = this.getKernelDisplayName(venvPath);
+
+        const venvProcessService = await this.processServiceFactory.create(undefined);
+        await venvProcessService.exec(
+            venvInterpreter.uri.fsPath,
+            [
+                '-m',
+                'ipykernel',
+                'install',
+                '--prefix',
+                venvPath.fsPath,
+                '--name',
+                kernelSpecName,
+                '--display-name',
+                kernelDisplayName
+            ],
+            { throwOnStdErr: false }
+        );
+
+        logger.info(`Kernel spec installed successfully to ${kernelSpecPath.fsPath}`);
+    }
+
     public getVenvHash(deepnoteFileUri: Uri): string {
         // Create a short hash from the file path for kernel naming and venv directory
         // This provides better uniqueness and prevents directory structure leakage

src/kernels/deepnote/environments/deepnoteEnvironmentManager.ts

@@ -191,13 +191,8 @@ export class DeepnoteEnvironmentManager implements IExtensionSyncActivationServi
             throw new Error(`Environment not found: ${id}`);
         }
 
-        if (config.serverInfo) {
-            logger.info(`Server already running for environment: ${config.name} (${id})`);
-            return;
-        }
-
         try {
-            logger.info(`Starting server for environment: ${config.name} (${id})`);
+            logger.info(`Ensuring server is running for environment: ${config.name} (${id})`);
 
             // First ensure venv is created and toolkit is installed
             await this.toolkitInstaller.ensureVenvAndToolkit(config.pythonInterpreter, config.venvPath, undefined);
@@ -207,7 +202,9 @@ export class DeepnoteEnvironmentManager implements IExtensionSyncActivationServi
                 await this.toolkitInstaller.installAdditionalPackages(config.venvPath, config.packages, undefined);
             }
 
-            // Start the Jupyter server
+            // Start the Jupyter server (serverStarter is idempotent - returns existing if running)
+            // IMPORTANT: Always call this to ensure we get the current server info
+            // Don't return early based on config.serverInfo - it may be stale!
             const serverInfo = await this.serverStarter.startServer(
                 config.pythonInterpreter,
                 config.venvPath,
@@ -221,7 +218,7 @@ export class DeepnoteEnvironmentManager implements IExtensionSyncActivationServi
             await this.persistEnvironments();
             this._onDidChangeEnvironments.fire();
 
-            logger.info(`Server started successfully for environment: ${config.name} (${id})`);
+            logger.info(`Server running for environment: ${config.name} (${id}) at ${serverInfo.url}`);
         } catch (error) {
             logger.error(`Failed to start server for environment: ${config.name} (${id})`, error);
             throw error;