Add a filesystem budget

This adds a wrapper around the 'fs/promises' module that will limit parallelism of filesystem operations. Larger projects (e.g. monorepos) have had crashes that look like file descriptor exhaustion.

src/caching/github-actions-cache.ts

@@ -12,7 +12,6 @@ import {createHash} from 'crypto';
 import {scriptReferenceToString} from '../config.js';
 import {getScriptDataDir} from '../util/script-data-dir.js';
 import {execFile} from 'child_process';
-import {createReadStream, createWriteStream} from 'fs';
 
 import type * as http from 'http';
 import type {Cache, CacheHit} from './cache.js';
@@ -248,7 +247,7 @@ export class GitHubActionsCache implements Cache {
         const end = offset + chunkSize - 1;
         offset += maxChunkSize;
 
-        const tarballChunkStream = createReadStream(tarballPath, {
+        const tarballChunkStream = await fs.createReadStream(tarballPath, {
           fd: tarballHandle.fd,
           start,
           end,
@@ -584,8 +583,8 @@ class GitHubActionsCacheHit implements CacheHit {
         `GitHub Cache download HTTP ${String(response.statusCode)} error`
       );
     }
+    const writeTarballStream = await fs.createWriteStream(tarballPath);
     await new Promise<void>((resolve, reject) => {
-      const writeTarballStream = createWriteStream(tarballPath);
       writeTarballStream.on('error', (error) => reject(error));
       response.on('error', (error) => reject(error));
       response.pipe(writeTarballStream);

src/caching/local-cache.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import * as fs from 'fs/promises';
+import * as fs from '../util/fs.js';
 import * as pathlib from 'path';
 import {createHash} from 'crypto';
 import {getScriptDataDir} from '../util/script-data-dir.js';

src/util/copy.ts

@@ -4,10 +4,9 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import * as fs from 'fs/promises';
+import * as fs from './fs.js';
 import * as pathlib from 'path';
 import {optimizeMkdirs} from './optimize-mkdirs.js';
-import {constants} from 'fs';
 import {IS_WINDOWS} from '../util/windows.js';
 
 import type {AbsoluteEntry} from './glob.js';
@@ -80,7 +79,7 @@ export const copyEntries = async (
  */
 const copyFileGracefully = async (src: string, dest: string): Promise<void> => {
   try {
-    await fs.copyFile(src, dest, constants.COPYFILE_EXCL);
+    await fs.copyFile(src, dest, fs.constants.COPYFILE_EXCL);
   } catch (error) {
     const {code} = error as {code: string};
     if (code === /* does not exist */ 'ENOENT') {

src/util/delete.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import * as fs from 'fs/promises';
+import * as fs from './fs.js';
 import * as pathlib from 'path';
 
 import type {AbsoluteEntry} from './glob.js';

src/util/fs.ts

@@ -8,6 +8,8 @@ import {
   createWriteStream as rawCreateWriteStream,
 } from 'fs';
 import {Deferred} from './deferred.js';
+export {constants} from 'fs';
+export type * from 'fs';
 
 declare global {
   interface SymbolConstructor {
@@ -25,7 +27,7 @@ interface Disposable {
 
 let maxOpenFiles = Number(process.env['WIREIT_MAX_OPEN_FILES']);
 if (isNaN(maxOpenFiles)) {
-  maxOpenFiles = 8000;
+  maxOpenFiles = 4000;
 }
 
 export function setMaxOpenFiles(n: number): void {
@@ -144,11 +146,29 @@ export async function access(path: string): Promise<void> {
   }
 }
 
+type ReadStreamOptions =
+  | BufferEncoding
+  | {
+      flags?: string | undefined;
+      encoding?: BufferEncoding | undefined;
+      fd?: number | undefined;
+      mode?: number | undefined;
+      autoClose?: boolean | undefined;
+      /**
+       * @default false
+       */
+      emitClose?: boolean | undefined;
+      start?: number | undefined;
+      end?: number | undefined;
+      highWaterMark?: number | undefined;
+    };
+
 export async function createReadStream(
-  path: string
+  path: string,
+  options?: ReadStreamOptions
 ): Promise<fsTypes.ReadStream> {
   const budget = await reserveFileBudget();
-  const stream = rawCreateReadStream(path);
+  const stream = rawCreateReadStream(path, options);
   stream.on('close', () => budget[Symbol.dispose]());
   return stream;
 }
@@ -161,3 +181,69 @@ export async function createWriteStream(
   stream.on('close', () => budget[Symbol.dispose]());
   return stream;
 }
+
+export async function copyFile(
+  src: fsTypes.PathLike,
+  dest: fsTypes.PathLike,
+  flags?: number | undefined
+) {
+  const budget = await reserveFileBudget();
+  try {
+    return await fs.copyFile(src, dest, flags);
+  } finally {
+    budget[Symbol.dispose]();
+  }
+}
+
+export function readlink(
+  path: fsTypes.PathLike,
+  options?: fsTypes.BaseEncodingOptions | BufferEncoding | null
+): Promise<string>;
+export function readlink(
+  path: fsTypes.PathLike,
+  options: fsTypes.BufferEncodingOption
+): Promise<Buffer>;
+export async function readlink(
+  path: fsTypes.PathLike,
+  options?:
+    | fsTypes.BaseEncodingOptions
+    | fsTypes.BufferEncodingOption
+    | string
+    | null
+): Promise<string | Buffer> {
+  const budget = await reserveFileBudget();
+  try {
+    return await fs.readlink(path, options as any);
+  } finally {
+    budget[Symbol.dispose]();
+  }
+}
+
+export async function symlink(target: fsTypes.PathLike, path: fsTypes.PathLike, type?: string|null) {
+const budget = await reserveFileBudget();
+try {
+  return await fs.symlink(target, path, type);
+} finally {
+  budget[Symbol.dispose]();
+}
+}
+
+export async function unlink(target: string) {
+const budget = await reserveFileBudget();
+try {
+  return await fs.unlink(target);
+} finally {
+  budget[Symbol.dispose]();
+}
+}
+
+export async function rmdir(target: string) {
+  const budget = await reserveFileBudget();
+  try {
+    return await fs.rmdir(target);
+  } finally {
+    budget[Symbol.dispose]();
+  }
+}
+
+

src/util/manifest.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import {Stats} from 'fs';
+import {Stats} from './fs.js';
 
 /**
  * Metadata about a file which we use as a heuristic to decide whether two files

src/util/package-json-reader.ts

@@ -8,7 +8,7 @@ import {Result} from '../error.js';
 import {AsyncCache} from './async-cache.js';
 import {PackageJson} from './package-json.js';
 import * as pathlib from 'path';
-import * as fs from 'fs/promises';
+import * as fs from './fs.js';
 import {parseTree} from './ast.js';
 
 export const astKey = Symbol('ast');