Skip to content

Commit

Permalink
Bump high water mark when downloading bundle to 16 MB
Browse files Browse the repository at this point in the history
  • Loading branch information
henrymercer committed Oct 22, 2024
1 parent 8c3a732 commit 06361b4
Show file tree
Hide file tree
Showing 6 changed files with 35 additions and 5 deletions.
3 changes: 3 additions & 0 deletions lib/tar.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lib/tar.js.map

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 9 additions & 1 deletion lib/tools-download.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lib/tools-download.js.map

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions src/tar.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,13 @@ export async function extractTarZst(
logger: Logger,
): Promise<string> {
const dest = await createExtractFolder();
logger.debug(
`Extracting to ${dest}.${
tar instanceof stream.Readable
? ` Input stream has high water mark ${tar.readableHighWaterMark}.`
: ""
}`,
);

try {
// Initialize args
Expand Down
16 changes: 14 additions & 2 deletions src/tools-download.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { IncomingMessage, OutgoingHttpHeaders } from "http";
import { IncomingMessage, OutgoingHttpHeaders, RequestOptions } from "http";
import * as path from "path";
import { performance } from "perf_hooks";

Expand All @@ -11,6 +11,11 @@ import { formatDuration, Logger } from "./logging";
import * as tar from "./tar";
import { cleanUpGlob } from "./util";

/**
* High watermark to use when streaming the download and extraction of the CodeQL tools.
*/
export const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB

/**
* Timing information for the download and extraction of the CodeQL tools when
* we fully download the bundle before extracting.
Expand Down Expand Up @@ -182,7 +187,14 @@ async function downloadAndExtractZstdWithStreaming(
headers,
);
const response = await new Promise<IncomingMessage>((resolve) =>
https.get(codeqlURL, { headers }, (r) => resolve(r)),
https.get(
codeqlURL,
{
headers,
highWaterMark: STREAMING_HIGH_WATERMARK_BYTES,
} as unknown as RequestOptions,
(r) => resolve(r),
),
);

if (response.statusCode !== 200) {
Expand Down

0 comments on commit 06361b4

Please sign in to comment.