[Bug?]: Typescript 5 Language server + pnp - No file descriptors available

[goloveychuk]

Self-service

• I'd be willing to implement a fix

Describe the bug

So, with yarn pnp enabled, we updated from 4.8.4 to 5.0.4 and now we're getting

2023-05-22 17:25:15.275 [error] Error: <semantic> TypeScript Server Error (5.0.4-sdk)
Can't open file: No file descriptors available
Libzip Error: Can't open file: No file descriptors available
    at ZipFS.makeLibzipError (/Users/vadymh/github/descriptors-repro/.pnp.cjs:1709:25)
    at new ZipFS (/Users/vadymh/github/descriptors-repro/.pnp.cjs:1684:20)
    at ZipOpenFS.getZipSync (/Users/vadymh/github/descriptors-repro/.pnp.cjs:3874:18)
    at ZipOpenFS.makeCallSync (/Users/vadymh/github/descriptors-repro/.pnp.cjs:3754:17)
    at ZipOpenFS.watchFile (/Users/vadymh/github/descriptors-repro/.pnp.cjs:3717:17)
    at VirtualFS.watchFile (/Users/vadymh/github/descriptors-repro/.pnp.cjs:2923:24)
    at PosixFS.watchFile (/Users/vadymh/github/descriptors-repro/.pnp.cjs:2923:24)
    at NodePathFS.watchFile (/Users/vadymh/github/descriptors-repro/.pnp.cjs:2923:24)
    at fsWatchFileWorker (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:8597:11)
    at /Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:8243:15
    at createSingleWatcherPerName (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:7764:16)
    at pollingWatchFile (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:8238:12)
    at watchFile2 (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:8073:16)
    at watchMissingFileSystemEntry (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:8331:14)
    at fsWatchHandlingExistenceOnHost (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:8262:72)
    at /Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:8252:15
    at createSingleWatcherPerName (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:7764:16)
    at fsWatch (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:8247:12)
    at Object.watchFile2 [as watchFile] (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:8098:16)
    at watchFile (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:117016:67)
    at Object.watchFile (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:117035:284)
    at InferredProject2.addMissingFileWatcher (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:172515:58)
    at /Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:172412:35
    at /Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:19717:21
    at Map.forEach (<anonymous>)
    at mutateMap (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:19715:10)
    at updateMissingFilePathsWatch (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:116908:3)
    at InferredProject2.updateGraphWorker (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:172408:7)
    at InferredProject2.updateGraph (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:172323:32)
    at _ProjectService.assignOrphanScriptInfoToInferredProject (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:174657:13)
    at _ProjectService.assignProjectToOpenedScriptInfo (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:176279:12)
    at /Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:176509:64
    at flatMap (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:2556:17)
    at _ProjectService.applyChangesInOpenFiles (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:176509:24)
    at updateOpen (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:177677:29)
    at /Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:180244:69
    at IpcIOSession.executeWithRequestId (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:180236:14)
    at IpcIOSession.executeCommand (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:180244:29)
    at IpcIOSession.onMessage (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:180286:51)
    at IpcIOSession.onMessage (/Users/vadymh/github/descriptors-repro/.yarn/sdks/typescript/lib/tsserver.js:201:32)
    at process.<anonymous> (/Users/vadymh/.yarn/berry/cache/typescript-patch-260000abbe-8.zip/node_modules/typescript/lib/tsserver.js:181861:14)
    at process.emit (node:events:513:28)
    at process.emit (/Users/vadymh/github/descriptors-repro/.pnp.cjs:9016:25)
    at emit (node:internal/child_process:958:14)
    at process.processTicksAndRejections (node:internal/process/task_queues:84:21)
    at Function.create (/Applications/Visual Studio Code.app/Contents/Resources/app/extensions/typescript-language-features/dist/extension.js:2:781356)
    at h.w (/Applications/Visual Studio Code.app/Contents/Resources/app/extensions/typescript-language-features/dist/extension.js:2:776555)
    at h.u (/Applications/Visual Studio Code.app/Contents/Resources/app/extensions/typescript-language-features/dist/extension.js:2:775620)
    at ChildProcess.<anonymous> (/Applications/Visual Studio Code.app/Contents/Resources/app/extensions/typescript-language-features/dist/extension.js:2:775252)
    at ChildProcess.emit (node:events:513:28)
    at emit (node:internal/child_process:958:14)
    at processTicksAndRejections (node:internal/process/task_queues:84:21)
2023-05-22 17:25:15.277 [error] Error: <semantic> TypeScript Server Error (5.0.4-sdk)

I've created a repro below, but I can't reproduce above error constantly. I got it once or twice, but reason could be other vscode open in parallel.
But what I can reproduce constantly, is that with 1k files (change it in gen.js) I have react-router resolved, but with 15k files it's not:

Error in this case is {"name":"Libzip Error","code":"ZIP_ER_OPEN"} (you can log in makeLibzipError, it's swallowed by ts server)

If you disable pnp and change sdk to default you will have react-router resolved for both 1k and 15k.

What I found that in ts 4.9 they changed default watch method
microsoft/TypeScript#50366
From what I see in source code (in node-server), ts language service is using now fs events method
UPD. Incorrect. Because stacktrace has fs.watchFile, which is polling under the hood.

One catch on systems which have limits on fs watchers like linux, it would still fallback to polling

From what I see, for both pnp and not watchFile is creating FD for every file.

To reproduce

git clone https://github.com/goloveychuk/yarn-repro
yarn install
node gen.js
open project in vscode
select correct sdk
open index.ts
reload vscode window

Environment

System:
    OS: macOS 12.6
    CPU: (16) x64 Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz
  Binaries:
    Node: 14.20.0 - /private/var/folders/_c/5f907xz5253bhss7ytr8m8zh0000gn/T/xfs-ac198389/node
    Yarn: 3.5.0 - /private/var/folders/_c/5f907xz5253bhss7ytr8m8zh0000gn/T/xfs-ac198389/yarn
    npm: 6.14.17 - /var/folders/_c/5f907xz5253bhss7ytr8m8zh0000gn/T/fnm_multishells/45914_1684763731174/bin/npm

Additional context

No response

[mredigonda]

Two related issues that were reported in VSCode itself:

• TS Server fatal error: Can't open file: No file descriptors available microsoft/vscode#184424
• TS Server fatal error: Can't open file: No file descriptors available microsoft/vscode#178527
• The JS/TS language service immediately crashed 5 times. The service will not be restarted. microsoft/vscode#182468 (comment)

I'm also experiencing this, using Typescript 4.9.4.

I have a monorepo with Yarn Workspaces, and sometimes, after having opened multiple files from different packages of my workspace, I get this error. It's quite frequent actually, and I'm starting to consider it a blocker.

I permanently increased max file descriptors (both soft and hard limits) but still the server fails after some reloads.

Let me know if you need more info, I could probably provide some logs. Also let me know of any workarounds, those would definitely be appreciated!

[goloveychuk]

Minimal repro.
@yarnpkg/libzip:2.3.0
Limit is near 4000 for mac os

const fs = require('fs');
const path = require('path')
const {execSync} = require('child_process')
const lipzip = require('@yarnpkg/libzip')

const count = 4300;

const rootDir = path.resolve(__dirname, 'gen');
fs.rmSync(rootDir, { recursive: true, force: true });

fs.mkdirSync(rootDir);


for (let i = 0; i < count; i++) {
    const p = path.join(rootDir, `file${i}.ts`);
    fs.writeFileSync(p, 'content')
    fs.openSync(p, 'r')
}

class Cls {
    libzip = lipzip.getLibzipSync()
    open(source) {
        const errPtr = this.libzip.malloc(4);
        try {
        let flags = 0;
        flags |= this.libzip.ZIP_RDONLY;
        this.readOnly = true;
        console.log('fd', fs.openSync(source, 'r'))
        this.zip = this.libzip.open(source, flags, errPtr);
        if (this.zip === 0) {
            const error = this.libzip.struct.errorS();
            this.libzip.error.initWithCode(error, this.libzip.getValue(errPtr, `i32`));
            throw this.makeLibzipError(error);
        }
        } finally {
            this.libzip.free(errPtr);
        }
    }
    makeLibzipError(error) {
        const errorCode = this.libzip.struct.errorCodeZip(error);
        const strerror = this.libzip.error.strerror(error);
        const libzipError = new Error(strerror+this.libzip.errors[errorCode]);
        return libzipError;
      }
}

setTimeout(() => {
    const zip = new Cls()
    zip.open('logs.txt.zip')
    console.log('ended')
}, 1000)




console.log('pid', process.pid)
setTimeout(() => {
    console.log('open descriptors: '+execSync(`lsof -p ${process.pid} | wc -l`, {encoding: 'utf8'}))
}, 3000)

setInterval(() => {
    
}, 1000);

just run it with node

[goloveychuk]

So, seems like node has bigger(ulimited) fds limit, zipfs is not.

It could be
https://github.com/nodejs/node/blob/cb7e854c776d366fed74ec3edf082aea0e723a6f/src/node.cc#L709

[goloveychuk]

One of the solutions which could work, we can open file fd with node and open zip with zip_fdopen
@merceyz wdyt?

[goloveychuk]

Wow. It's fixed in master

berry/packages/yarnpkg-libzip/sources/ZipFS.ts

Line 162 in ad03119

const lzSource = this.allocateUnattachedSource(source);

1431217#diff-523d93f5ae8070efae5918f815cd1be6ecd605cc6963491e8732dbe46bf316d7L161

[goloveychuk]

it seems like yarn set version from sources fixing an issue.
Bundle version: 4.0.0-rc.45.git.20230607.hash-ad03119

[arcanis]

Did you reproduce it since then? Marking the thread as auto-closable.

[goloveychuk]

No, highly confident.