Skip to content

Commit 53e6d47

Browse files
ericallamericallam
authored
Fix deploy timeout issues (#1661)
* deploy v2 streaming WIP * finalize deployment now SSE and won't timeout * handle zodfetchSSE connection errors * Create tender-cycles-melt.md
1 parent 7fa45ad commit 53e6d47

File tree

11 files changed

+339
-43
lines changed

11 files changed

+339
-43
lines changed

‎.changeset/tender-cycles-melt.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"trigger.dev": patch
3+
"@trigger.dev/core": patch
4+
---
5+
6+
Fixed deploy timeout issues and improve the output of logs when deploying

‎apps/webapp/app/routes/api.v2.deployments.$deploymentId.finalize.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { z } from "zod";
44
import { authenticateApiRequest } from "~/services/apiAuth.server";
55
import { logger } from "~/services/logger.server";
66
import { ServiceValidationError } from "~/v3/services/baseService.server";
7-
import { FinalizeDeploymentV2Service } from "~/v3/services/finalizeDeploymentV2";
7+
import { FinalizeDeploymentV2Service } from "~/v3/services/finalizeDeploymentV2.server";
88

99
const ParamsSchema = z.object({
1010
deploymentId: z.string(),

‎apps/webapp/app/routes/api.v3.deployments.$deploymentId.finalize.ts

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
2+
import { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3";
3+
import { z } from "zod";
4+
import { authenticateApiRequest } from "~/services/apiAuth.server";
5+
import { logger } from "~/services/logger.server";
6+
import { ServiceValidationError } from "~/v3/services/baseService.server";
7+
import { FinalizeDeploymentV2Service } from "~/v3/services/finalizeDeploymentV2.server";
8+
9+
const ParamsSchema = z.object({
10+
deploymentId: z.string(),
11+
});
12+
13+
export async function action({ request, params }: ActionFunctionArgs) {
14+
// Ensure this is a POST request
15+
if (request.method.toUpperCase() !== "POST") {
16+
return { status: 405, body: "Method Not Allowed" };
17+
}
18+
19+
const parsedParams = ParamsSchema.safeParse(params);
20+
21+
if (!parsedParams.success) {
22+
return json({ error: "Invalid params" }, { status: 400 });
23+
}
24+
25+
// Next authenticate the request
26+
const authenticationResult = await authenticateApiRequest(request);
27+
28+
if (!authenticationResult) {
29+
logger.info("Invalid or missing api key", { url: request.url });
30+
return json({ error: "Invalid or Missing API key" }, { status: 401 });
31+
}
32+
33+
const authenticatedEnv = authenticationResult.environment;
34+
35+
const { deploymentId } = parsedParams.data;
36+
37+
const rawBody = await request.json();
38+
const body = FinalizeDeploymentRequestBody.safeParse(rawBody);
39+
40+
if (!body.success) {
41+
return json({ error: "Invalid body", issues: body.error.issues }, { status: 400 });
42+
}
43+
44+
try {
45+
// Create a text stream chain
46+
const stream = new TransformStream();
47+
const encoder = new TextEncoderStream();
48+
const writer = stream.writable.getWriter();
49+
50+
const service = new FinalizeDeploymentV2Service();
51+
52+
// Chain the streams: stream -> encoder -> response
53+
const response = new Response(stream.readable.pipeThrough(encoder), {
54+
headers: {
55+
"Content-Type": "text/event-stream",
56+
"Cache-Control": "no-cache",
57+
Connection: "keep-alive",
58+
},
59+
});
60+
61+
const pingInterval = setInterval(() => {
62+
writer.write("event: ping\ndata: {}\n\n");
63+
}, 10000); // 10 seconds
64+
65+
service
66+
.call(authenticatedEnv, deploymentId, body.data, writer)
67+
.then(async () => {
68+
clearInterval(pingInterval);
69+
70+
await writer.write(`event: complete\ndata: ${JSON.stringify({ id: deploymentId })}\n\n`);
71+
await writer.close();
72+
})
73+
.catch(async (error) => {
74+
let errorMessage;
75+
76+
if (error instanceof ServiceValidationError) {
77+
errorMessage = { error: error.message };
78+
} else if (error instanceof Error) {
79+
logger.error("Error finalizing deployment", { error: error.message });
80+
errorMessage = { error: `Internal server error: ${error.message}` };
81+
} else {
82+
logger.error("Error finalizing deployment", { error: String(error) });
83+
errorMessage = { error: "Internal server error" };
84+
}
85+
86+
clearInterval(pingInterval);
87+
88+
await writer.write(`event: error\ndata: ${JSON.stringify(errorMessage)}\n\n`);
89+
await writer.close();
90+
});
91+
92+
return response;
93+
} catch (error) {
94+
if (error instanceof ServiceValidationError) {
95+
return json({ error: error.message }, { status: 400 });
96+
} else if (error instanceof Error) {
97+
logger.error("Error finalizing deployment", { error: error.message });
98+
return json({ error: `Internal server error: ${error.message}` }, { status: 500 });
99+
} else {
100+
logger.error("Error finalizing deployment", { error: String(error) });
101+
return json({ error: "Internal server error" }, { status: 500 });
102+
}
103+
}
104+
}

‎apps/webapp/app/v3/services/finalizeDeployment.server.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,12 @@ export class FinalizeDeploymentService extends BaseService {
4343
throw new ServiceValidationError("Worker deployment does not have a worker");
4444
}
4545

46+
if (deployment.status === "DEPLOYED") {
47+
logger.debug("Worker deployment is already deployed", { id });
48+
49+
return deployment;
50+
}
51+
4652
if (deployment.status !== "DEPLOYING") {
4753
logger.error("Worker deployment is not in DEPLOYING status", { id });
4854
throw new ServiceValidationError("Worker deployment is not in DEPLOYING status");

‎apps/webapp/app/v3/services/finalizeDeploymentV2.ts renamed to ‎apps/webapp/app/v3/services/finalizeDeploymentV2.server.ts

Lines changed: 34 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ export class FinalizeDeploymentV2Service extends BaseService {
1313
public async call(
1414
authenticatedEnv: AuthenticatedEnvironment,
1515
id: string,
16-
body: FinalizeDeploymentRequestBody
16+
body: FinalizeDeploymentRequestBody,
17+
writer?: WritableStreamDefaultWriter
1718
) {
1819
// if it's self hosted, lets just use the v1 finalize deployment service
1920
if (body.selfHosted) {
@@ -83,24 +84,27 @@ export class FinalizeDeploymentV2Service extends BaseService {
8384
throw new ServiceValidationError("Missing depot token");
8485
}
8586

86-
const pushResult = await executePushToRegistry({
87-
depot: {
88-
buildId: externalBuildData.data.buildId,
89-
orgToken: env.DEPOT_TOKEN,
90-
projectId: externalBuildData.data.projectId,
91-
},
92-
registry: {
93-
host: env.DEPLOY_REGISTRY_HOST,
94-
namespace: env.DEPLOY_REGISTRY_NAMESPACE,
95-
username: env.DEPLOY_REGISTRY_USERNAME,
96-
password: env.DEPLOY_REGISTRY_PASSWORD,
97-
},
98-
deployment: {
99-
version: deployment.version,
100-
environmentSlug: deployment.environment.slug,
101-
projectExternalRef: deployment.worker.project.externalRef,
87+
const pushResult = await executePushToRegistry(
88+
{
89+
depot: {
90+
buildId: externalBuildData.data.buildId,
91+
orgToken: env.DEPOT_TOKEN,
92+
projectId: externalBuildData.data.projectId,
93+
},
94+
registry: {
95+
host: env.DEPLOY_REGISTRY_HOST,
96+
namespace: env.DEPLOY_REGISTRY_NAMESPACE,
97+
username: env.DEPLOY_REGISTRY_USERNAME,
98+
password: env.DEPLOY_REGISTRY_PASSWORD,
99+
},
100+
deployment: {
101+
version: deployment.version,
102+
environmentSlug: deployment.environment.slug,
103+
projectExternalRef: deployment.worker.project.externalRef,
104+
},
102105
},
103-
});
106+
writer
107+
);
104108

105109
if (!pushResult.ok) {
106110
throw new ServiceValidationError(pushResult.error);
@@ -148,11 +152,10 @@ type ExecutePushResult =
148152
logs: string;
149153
};
150154

151-
async function executePushToRegistry({
152-
depot,
153-
registry,
154-
deployment,
155-
}: ExecutePushToRegistryOptions): Promise<ExecutePushResult> {
155+
async function executePushToRegistry(
156+
{ depot, registry, deployment }: ExecutePushToRegistryOptions,
157+
writer?: WritableStreamDefaultWriter
158+
): Promise<ExecutePushResult> {
156159
// Step 1: We need to "login" to the digital ocean registry
157160
const configDir = await ensureLoggedIntoDockerRegistry(registry.host, {
158161
username: registry.username,
@@ -180,7 +183,7 @@ async function executePushToRegistry({
180183
try {
181184
const processCode = await new Promise<number | null>((res, rej) => {
182185
// For some reason everything is output on stderr, not stdout
183-
childProcess.stderr?.on("data", (data: Buffer) => {
186+
childProcess.stderr?.on("data", async (data: Buffer) => {
184187
const text = data.toString();
185188

186189
// Emitted data chunks can contain multiple lines. Remove empty lines.
@@ -191,6 +194,13 @@ async function executePushToRegistry({
191194
imageTag,
192195
deployment,
193196
});
197+
198+
// Now we can write strings directly
199+
if (writer) {
200+
for (const line of lines) {
201+
await writer.write(`event: log\ndata: ${JSON.stringify({ message: line })}\n\n`);
202+
}
203+
}
194204
});
195205

196206
childProcess.on("error", (e) => rej(e));

‎packages/cli-v3/src/apiClient.ts

Lines changed: 57 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import {
2121
FailDeploymentResponseBody,
2222
FinalizeDeploymentRequestBody,
2323
} from "@trigger.dev/core/v3";
24-
import { zodfetch, ApiError } from "@trigger.dev/core/v3/zodfetch";
24+
import { zodfetch, ApiError, zodfetchSSE } from "@trigger.dev/core/v3/zodfetch";
2525

2626
export class CliApiClient {
2727
constructor(
@@ -247,23 +247,72 @@ export class CliApiClient {
247247
);
248248
}
249249

250-
async finalizeDeployment(id: string, body: FinalizeDeploymentRequestBody) {
250+
async finalizeDeployment(
251+
id: string,
252+
body: FinalizeDeploymentRequestBody,
253+
onLog?: (message: string) => void
254+
): Promise<ApiResult<FailDeploymentResponseBody>> {
251255
if (!this.accessToken) {
252256
throw new Error("finalizeDeployment: No access token");
253257
}
254258

255-
return wrapZodFetch(
256-
FailDeploymentResponseBody,
257-
`${this.apiURL}/api/v2/deployments/${id}/finalize`,
258-
{
259+
let resolvePromise: (value: ApiResult<FailDeploymentResponseBody>) => void;
260+
let rejectPromise: (reason: any) => void;
261+
262+
const promise = new Promise<ApiResult<FailDeploymentResponseBody>>((resolve, reject) => {
263+
resolvePromise = resolve;
264+
rejectPromise = reject;
265+
});
266+
267+
const source = zodfetchSSE({
268+
url: `${this.apiURL}/api/v3/deployments/${id}/finalize`,
269+
request: {
259270
method: "POST",
260271
headers: {
261272
Authorization: `Bearer ${this.accessToken}`,
262273
"Content-Type": "application/json",
263274
},
264275
body: JSON.stringify(body),
265-
}
266-
);
276+
},
277+
messages: {
278+
error: z.object({ error: z.string() }),
279+
log: z.object({ message: z.string() }),
280+
complete: FailDeploymentResponseBody,
281+
},
282+
});
283+
284+
source.onConnectionError((error) => {
285+
rejectPromise({
286+
success: false,
287+
error,
288+
});
289+
});
290+
291+
source.onMessage("complete", (message) => {
292+
resolvePromise({
293+
success: true,
294+
data: message,
295+
});
296+
});
297+
298+
source.onMessage("error", ({ error }) => {
299+
rejectPromise({
300+
success: false,
301+
error,
302+
});
303+
});
304+
305+
if (onLog) {
306+
source.onMessage("log", ({ message }) => {
307+
onLog(message);
308+
});
309+
}
310+
311+
const result = await promise;
312+
313+
source.stop();
314+
315+
return result;
267316
}
268317

269318
async startDeploymentIndexing(deploymentId: string, body: StartDeploymentIndexingRequestBody) {

‎packages/cli-v3/src/commands/deploy.ts

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -222,10 +222,10 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
222222
forcedExternals,
223223
listener: {
224224
onBundleStart() {
225-
$buildSpinner.start("Building project");
225+
$buildSpinner.start("Building trigger code");
226226
},
227227
onBundleComplete(result) {
228-
$buildSpinner.stop("Successfully built project");
228+
$buildSpinner.stop("Successfully built code");
229229

230230
logger.debug("Bundle result", result);
231231
},
@@ -328,9 +328,9 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
328328
const $spinner = spinner();
329329

330330
if (isLinksSupported) {
331-
$spinner.start(`Deploying version ${version} ${deploymentLink}`);
331+
$spinner.start(`Building version ${version} ${deploymentLink}`);
332332
} else {
333-
$spinner.start(`Deploying version ${version}`);
333+
$spinner.start(`Building version ${version}`);
334334
}
335335

336336
const selfHostedRegistryHost = deployment.registryHost ?? options.registry;
@@ -359,6 +359,13 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
359359
compilationPath: destination.path,
360360
buildEnvVars: buildManifest.build.env,
361361
network: options.network,
362+
onLog: (logMessage) => {
363+
if (isLinksSupported) {
364+
$spinner.message(`Building version ${version} ${deploymentLink}: ${logMessage}`);
365+
} else {
366+
$spinner.message(`Building version ${version}: ${logMessage}`);
367+
}
368+
},
362369
});
363370

364371
logger.debug("Build result", buildResult);
@@ -426,10 +433,26 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
426433
}`
427434
: `${buildResult.image}${buildResult.digest ? `@${buildResult.digest}` : ""}`;
428435

429-
const finalizeResponse = await projectClient.client.finalizeDeployment(deployment.id, {
430-
imageReference,
431-
selfHosted: options.selfHosted,
432-
});
436+
if (isLinksSupported) {
437+
$spinner.message(`Deploying version ${version} ${deploymentLink}`);
438+
} else {
439+
$spinner.message(`Deploying version ${version}`);
440+
}
441+
442+
const finalizeResponse = await projectClient.client.finalizeDeployment(
443+
deployment.id,
444+
{
445+
imageReference,
446+
selfHosted: options.selfHosted,
447+
},
448+
(logMessage) => {
449+
if (isLinksSupported) {
450+
$spinner.message(`Deploying version ${version} ${deploymentLink}: ${logMessage}`);
451+
} else {
452+
$spinner.message(`Deploying version ${version}: ${logMessage}`);
453+
}
454+
}
455+
);
433456

434457
if (!finalizeResponse.success) {
435458
await failDeploy(

0 commit comments

Comments
 (0)