From f1fd62a11642de45eb87ebacb044fe8fcf2beea2 Mon Sep 17 00:00:00 2001 From: Greg Brimble Date: Wed, 2 Nov 2022 11:26:31 +0000 Subject: [PATCH] Avoid uploading the 'functions' directory as part of 'wrangler pages publish' (#2103) --- .changeset/cool-nails-marry.md | 9 ++ package-lock.json | 41 +++++++ packages/wrangler/package.json | 1 + packages/wrangler/src/__tests__/pages.test.ts | 110 ++++++++++++++++++ packages/wrangler/src/is-interactive.ts | 4 + packages/wrangler/src/pages/constants.ts | 2 + packages/wrangler/src/pages/upload.tsx | 32 +++-- 7 files changed, 187 insertions(+), 12 deletions(-) create mode 100644 .changeset/cool-nails-marry.md diff --git a/.changeset/cool-nails-marry.md b/.changeset/cool-nails-marry.md new file mode 100644 index 000000000000..2f2f2886f22d --- /dev/null +++ b/.changeset/cool-nails-marry.md @@ -0,0 +1,9 @@ +--- +"wrangler": patch +--- + +fix: Don't upload `functions/` directory as part of `wrangler pages publish` + +If the root directory of a project was the same as the build output directory, we were previously uploading the `functions/` directory as static assets. This PR now ensures that the `functions/` files are only used to create Pages Functions and are no longer uploaded as static assets. + +Additionally, we also now _do_ upload `_worker.js`, `_headers`, `_redirects` and `_routes.json` if they aren't immediate children of the build output directory. Previously, we'd ignore all files with this name regardless of location. For example, if you have a `public/blog/how-to-use-pages/_headers` file (where `public` is your build output directory), we will now upload the `_headers` file as a static asset. diff --git a/package-lock.json b/package-lock.json index 1df1434fa1b0..b13ce78e4908 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21848,6 +21848,7 @@ "jest-fetch-mock": "^3.0.3", "jest-websocket-mock": "^2.3.0", "mime": "^3.0.0", + "minimatch": "^5.1.0", "msw": "^0.47.1", "npx-import": "^1.1.3", "open": "^8.4.0", @@ -21890,6 +21891,15 @@ "dev": true, "license": "ISC" }, + "packages/wrangler/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, "packages/wrangler/node_modules/dotenv": { "version": "16.0.0", "dev": true, @@ -21927,6 +21937,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "packages/wrangler/node_modules/minimatch": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.0.tgz", + "integrity": "sha512-9TPBGGak4nHfGZsPBohm9AWg6NoT7QTCehS3BIJABslyZbzxfV78QM2Y6+i741OPZIafFAaiiEMh5OyIrJPgtg==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "packages/wrangler/node_modules/p-limit": { "version": "4.0.0", "dev": true, @@ -37264,6 +37286,7 @@ "jest-websocket-mock": "^2.3.0", "mime": "^3.0.0", "miniflare": "2.10.0", + "minimatch": "*", "msw": "^0.47.1", "nanoid": "^3.3.3", "npx-import": "^1.1.3", @@ -37296,6 +37319,15 @@ "version": "3.0.0", "dev": true }, + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0" + } + }, "dotenv": { "version": "16.0.0", "dev": true @@ -37315,6 +37347,15 @@ "p-locate": "^6.0.0" } }, + "minimatch": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.0.tgz", + "integrity": "sha512-9TPBGGak4nHfGZsPBohm9AWg6NoT7QTCehS3BIJABslyZbzxfV78QM2Y6+i741OPZIafFAaiiEMh5OyIrJPgtg==", + "dev": true, + "requires": { + "brace-expansion": "^2.0.1" + } + }, "p-limit": { "version": "4.0.0", "dev": true, diff --git a/packages/wrangler/package.json b/packages/wrangler/package.json index 5318c2318e32..5337307b658c 100644 --- a/packages/wrangler/package.json +++ b/packages/wrangler/package.json @@ -160,6 +160,7 @@ "jest-fetch-mock": "^3.0.3", "jest-websocket-mock": "^2.3.0", "mime": "^3.0.0", + "minimatch": "^5.1.0", "msw": "^0.47.1", "npx-import": "^1.1.3", "open": "^8.4.0", diff --git a/packages/wrangler/src/__tests__/pages.test.ts b/packages/wrangler/src/__tests__/pages.test.ts index f765aaa75b59..8019c179d5f0 100644 --- a/packages/wrangler/src/__tests__/pages.test.ts +++ b/packages/wrangler/src/__tests__/pages.test.ts @@ -1905,6 +1905,116 @@ and that at least one include rule is provided. `); }); + it("should avoid uploading some files", async () => { + mkdirSync("some_dir/node_modules", { recursive: true }); + mkdirSync("some_dir/functions", { recursive: true }); + + writeFileSync("logo.png", "foobar"); + writeFileSync("some_dir/functions/foo.js", "func"); + writeFileSync("some_dir/_headers", "headersfile"); + + writeFileSync("_headers", "headersfile"); + writeFileSync("_redirects", "redirectsfile"); + writeFileSync("_worker.js", "workerfile"); + writeFileSync("_routes.json", "routesfile"); + mkdirSync(".git"); + writeFileSync(".git/foo", "gitfile"); + writeFileSync("some_dir/node_modules/some_package", "nodefile"); + mkdirSync("functions"); + writeFileSync("functions/foo.js", "func"); + + setMockResponse( + "/pages/assets/check-missing", + "POST", + async (_, init) => { + const body = JSON.parse(init.body as string) as { hashes: string[] }; + assertLater(() => { + expect(init.headers).toMatchObject({ + Authorization: "Bearer <>", + }); + expect(body).toMatchObject({ + hashes: [ + "2082190357cfd3617ccfe04f340c6247", + "95dedb64e6d4940fc2e0f11f711cc2f4", + "09a79777abda8ccc8bdd51dd3ff8e9e9", + ], + }); + }); + return body.hashes; + } + ); + + // Accumulate multiple requests then assert afterwards + const requests: RequestInit[] = []; + setMockRawResponse("/pages/assets/upload", "POST", async (_, init) => { + requests.push(init); + + return createFetchResult(null, true); + }); + + assertLater(() => { + expect(requests.length).toBe(3); + + expect(requests[0].headers).toMatchObject({ + Authorization: "Bearer <>", + }); + + let body = JSON.parse( + requests[0].body as string + ) as UploadPayloadFile[]; + expect(body).toMatchObject([ + { + key: "95dedb64e6d4940fc2e0f11f711cc2f4", + value: Buffer.from("headersfile").toString("base64"), + metadata: { + contentType: "application/octet-stream", + }, + base64: true, + }, + ]); + + expect(requests[1].headers).toMatchObject({ + Authorization: "Bearer <>", + }); + + body = JSON.parse(requests[1].body as string) as UploadPayloadFile[]; + expect(body).toMatchObject([ + { + key: "2082190357cfd3617ccfe04f340c6247", + value: Buffer.from("foobar").toString("base64"), + metadata: { + contentType: "image/png", + }, + base64: true, + }, + ]); + + expect(requests[2].headers).toMatchObject({ + Authorization: "Bearer <>", + }); + + body = JSON.parse(requests[2].body as string) as UploadPayloadFile[]; + expect(body).toMatchObject([ + { + key: "09a79777abda8ccc8bdd51dd3ff8e9e9", + value: Buffer.from("func").toString("base64"), + metadata: { + contentType: "application/javascript", + }, + base64: true, + }, + ]); + }); + + await runWrangler("pages project upload ."); + + expect(std.out).toMatchInlineSnapshot(` + "✨ Success! Uploaded 3 files (TIMINGS) + + ✨ Upload complete!" + `); + }); + it("should retry uploads", async () => { writeFileSync("logo.txt", "foobar"); diff --git a/packages/wrangler/src/is-interactive.ts b/packages/wrangler/src/is-interactive.ts index fa9ef5795be6..2924fbf28084 100644 --- a/packages/wrangler/src/is-interactive.ts +++ b/packages/wrangler/src/is-interactive.ts @@ -4,6 +4,10 @@ * or you're piping values from / to another process, etc */ export default function isInteractive(): boolean { + if (process.env.CF_PAGES === "1") { + return false; + } + try { return Boolean(process.stdin.isTTY && process.stdout.isTTY); } catch { diff --git a/packages/wrangler/src/pages/constants.ts b/packages/wrangler/src/pages/constants.ts index 899c8405065f..0ed15fa10053 100644 --- a/packages/wrangler/src/pages/constants.ts +++ b/packages/wrangler/src/pages/constants.ts @@ -1,5 +1,7 @@ import { version as wranglerVersion } from "../../package.json"; +export const MAX_ASSET_COUNT = 20_000; +export const MAX_ASSET_SIZE = 25 * 1024 * 1024; export const PAGES_CONFIG_CACHE_FILENAME = "pages.json"; export const MAX_BUCKET_SIZE = 50 * 1024 * 1024; export const MAX_BUCKET_FILE_COUNT = 5000; diff --git a/packages/wrangler/src/pages/upload.tsx b/packages/wrangler/src/pages/upload.tsx index 05778749180c..75065665d914 100644 --- a/packages/wrangler/src/pages/upload.tsx +++ b/packages/wrangler/src/pages/upload.tsx @@ -3,13 +3,17 @@ import { dirname, join, relative, resolve, sep } from "node:path"; import { render, Text } from "ink"; import Spinner from "ink-spinner"; import { getType } from "mime"; +import { Minimatch } from "minimatch"; import PQueue from "p-queue"; import prettyBytes from "pretty-bytes"; import React from "react"; import { fetchResult } from "../cfetch"; import { FatalError } from "../errors"; +import isInteractive from "../is-interactive"; import { logger } from "../logger"; import { + MAX_ASSET_COUNT, + MAX_ASSET_SIZE, BULK_UPLOAD_CONCURRENCY, MAX_BUCKET_FILE_COUNT, MAX_BUCKET_SIZE, @@ -96,10 +100,11 @@ export const upload = async ( "_redirects", "_headers", "_routes.json", - ".DS_Store", - "node_modules", - ".git", - ]; + "functions", + "**/.DS_Store", + "**/node_modules", + "**/.git", + ].map((pattern) => new Minimatch(pattern)); const directory = resolve(args.directory); @@ -121,10 +126,13 @@ export const upload = async ( await Promise.all( files.map(async (file) => { const filepath = join(dir, file); + const relativeFilepath = relative(startingDir, filepath); const filestat = await stat(filepath); - if (IGNORE_LIST.includes(file)) { - return; + for (const minimatch of IGNORE_LIST) { + if (minimatch.match(relativeFilepath)) { + return; + } } if (filestat.isSymbolicLink()) { @@ -134,12 +142,12 @@ export const upload = async ( if (filestat.isDirectory()) { fileMap = await walk(filepath, fileMap, startingDir); } else { - const name = relative(startingDir, filepath).split(sep).join("/"); + const name = relativeFilepath.split(sep).join("/"); - if (filestat.size > 25 * 1024 * 1024) { + if (filestat.size > MAX_ASSET_SIZE) { throw new FatalError( `Error: Pages only supports files up to ${prettyBytes( - 25 * 1024 * 1024 + MAX_ASSET_SIZE )} in size\n${name} is ${prettyBytes(filestat.size)} in size`, 1 ); @@ -161,9 +169,9 @@ export const upload = async ( const fileMap = await walk(directory); - if (fileMap.size > 20000) { + if (fileMap.size > MAX_ASSET_COUNT) { throw new FatalError( - `Error: Pages only supports up to 20,000 files in a deployment. Ensure you have specified your build output directory correctly.`, + `Error: Pages only supports up to ${MAX_ASSET_COUNT.toLocaleString()} files in a deployment. Ensure you have specified your build output directory correctly.`, 1 ); } @@ -390,7 +398,7 @@ function Progress({ done, total }: { done: number; total: number }) { return ( <> - + {isInteractive() ? : null} {` Uploading... (${done}/${total})\n`}