Skip to content

Commit 2d8a41b

Browse files
ericallamericallam
authored
feat: realtime (#1402)
* Denormalize run tags, increase character limit to 128 * WIP realtime subscribing to runs * extracted the stream stuff into core, made it more reusable * WIP tags * Remove tags for now because it’s not support in electric * Support async iterables, readable stream, and callback style subscription styles * Remove tags streaming endpoint * Add realtime rate limits and scope them to the /realtime path * WIP rate limt per org * Introduce per org rate limits * WIP JWT auth * Move migrations into new internal db package * Resolve pnpm lock file * Authenticating to the realtime API with JWTs are working * realtime in the client * Created react-hooks package and starting to move stuff in there * Improve types for hooks * schema tasks * Added useBatch hook * build uploadthing/fal demo and change how run metadata is synced to the server * tweaks * WIL realtime concurrency tracking * Implement test for realtime client using testcontainers also updated electric to latest version * Allow customizing the expiration time of the automatic JWT created after triggering a task * Add support for subscribing to run tags * Improve auth types and API * finalize the realtime API * Fixed some example stuff * Allow up to 10 run tags * Remove core from docker-provider tsconfig paths to prevent it from being typechecked * do the same for the kubernetes provider * Fixing some typecheck errors * Fix webapp type errors * Update @trigger.dev/platform to 1.0.13 * Fix attw error * Remove from/to in subscribeToRuns query params * Add tests for the rate limit middleware and add custom JWT rate limits * turn off webapp test parallelism * Finish renaming jwt -> publicAccessToken and automatically give the JWT read access to the tags when using trigger * Add changeset * Attempt to fix unit tests in CI * Skip running the auth rate limit middleware tests for now * Try a beefier machine * Try and run webapp tests separately * Setup env vars * Make sliding window test more reliabile
1 parent 67542a5 commit 2d8a41b

File tree

153 files changed

+7921
-1522
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

153 files changed

+7921
-1522
lines changed

‎.changeset/brave-forks-compare.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
"@trigger.dev/react-hooks": minor
3+
"@trigger.dev/sdk": minor
4+
"@trigger.dev/core": minor
5+
---
6+
7+
Access run status updates in realtime, from your server or from your frontend

‎.github/workflows/release.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ concurrency:
1717
jobs:
1818
release:
1919
name: 🦋 Changesets Release
20-
runs-on: buildjet-8vcpu-ubuntu-2204
20+
runs-on: ubuntu-latest
2121
if: github.repository == 'triggerdotdev/trigger.dev'
2222
outputs:
2323
published: ${{ steps.changesets.outputs.published }}

‎.github/workflows/unit-tests.yml

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ on:
66
jobs:
77
unitTests:
88
name: "🧪 Unit Tests"
9-
runs-on: buildjet-8vcpu-ubuntu-2204
9+
runs-on: buildjet-16vcpu-ubuntu-2204
1010
steps:
1111
- name: ⬇️ Checkout repo
1212
uses: actions/checkout@v4
@@ -30,5 +30,15 @@ jobs:
3030
- name: 📀 Generate Prisma Client
3131
run: pnpm run generate
3232

33-
- name: 🧪 Run Unit Tests
34-
run: pnpm run test
33+
- name: 🧪 Run Webapp Unit Tests
34+
run: pnpm run test --filter webapp
35+
env:
36+
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
37+
DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
38+
SESSION_SECRET: "secret"
39+
MAGIC_LINK_SECRET: "secret"
40+
ENCRYPTION_KEY: "secret"
41+
42+
43+
- name: 🧪 Run Internal Unit Tests
44+
run: pnpm run test --filter "@internal/*"

‎.npmrc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
link-workspace-packages=false
2-
public-hoist-pattern[]=*prisma*
2+
public-hoist-pattern[]=*prisma*
3+
prefer-workspace-packages=true

‎.vscode/extensions.json

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,4 @@
11
{
2-
"recommendations": [
3-
"denoland.vscode-deno"
4-
],
5-
"unwantedRecommendations": [
6-
7-
]
2+
"recommendations": ["bierner.comment-tagged-templates"],
3+
"unwantedRecommendations": []
84
}

‎apps/docker-provider/tsconfig.json

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,6 @@
66
"forceConsistentCasingInFileNames": true,
77
"resolveJsonModule": true,
88
"strict": true,
9-
"skipLibCheck": true,
10-
"paths": {
11-
"@trigger.dev/core/v3": ["../../packages/core/src/v3"],
12-
"@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
13-
}
9+
"skipLibCheck": true
1410
}
1511
}

‎apps/kubernetes-provider/tsconfig.json

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,6 @@
66
"forceConsistentCasingInFileNames": true,
77
"resolveJsonModule": true,
88
"strict": true,
9-
"skipLibCheck": true,
10-
"paths": {
11-
"@trigger.dev/core/v3": ["../../packages/core/src/v3"],
12-
"@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
13-
}
9+
"skipLibCheck": true
1410
}
1511
}

‎apps/webapp/app/env.server.ts

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ const EnvironmentSchema = z.object({
3131
REMIX_APP_PORT: z.string().optional(),
3232
LOGIN_ORIGIN: z.string().default("http://localhost:3030"),
3333
APP_ORIGIN: z.string().default("http://localhost:3030"),
34-
ELECTRIC_ORIGIN: z.string(),
34+
ELECTRIC_ORIGIN: z.string().default("http://localhost:3060"),
3535
APP_ENV: z.string().default(process.env.NODE_ENV),
3636
SERVICE_NAME: z.string().default("trigger.dev webapp"),
3737
SECRET_STORE: SecretStoreOptionsSchema.default("DATABASE"),
@@ -103,6 +103,25 @@ const EnvironmentSchema = z.object({
103103
API_RATE_LIMIT_REFILL_RATE: z.coerce.number().int().default(250), // refix 250 tokens every 10 seconds
104104
API_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"),
105105
API_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"),
106+
API_RATE_LIMIT_LIMITER_LOGS_ENABLED: z.string().default("0"),
107+
108+
API_RATE_LIMIT_JWT_WINDOW: z.string().default("1m"),
109+
API_RATE_LIMIT_JWT_TOKENS: z.coerce.number().int().default(60),
110+
111+
//Realtime rate limiting
112+
/**
113+
* @example "60s"
114+
* @example "1m"
115+
* @example "1h"
116+
* @example "1d"
117+
* @example "1000ms"
118+
* @example "1000s"
119+
*/
120+
REALTIME_RATE_LIMIT_WINDOW: z.string().default("1m"),
121+
REALTIME_RATE_LIMIT_TOKENS: z.coerce.number().int().default(100),
122+
REALTIME_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"),
123+
REALTIME_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"),
124+
REALTIME_RATE_LIMIT_LIMITER_LOGS_ENABLED: z.string().default("0"),
106125

107126
//Ingesting event rate limit
108127
INGEST_EVENT_RATE_LIMIT_WINDOW: z.string().default("60s"),

‎apps/webapp/app/models/taskRunTag.server.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { prisma } from "~/db.server";
22
import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
33

4-
export const MAX_TAGS_PER_RUN = 5;
4+
export const MAX_TAGS_PER_RUN = 10;
55

66
export async function createTag({ tag, projectId }: { tag: string; projectId: string }) {
77
if (tag.trim().length === 0) return;

‎apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts

Lines changed: 44 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,7 @@ type CommonRelatedRun = Prisma.Result<
6262
export class ApiRetrieveRunPresenter extends BasePresenter {
6363
public async call(
6464
friendlyId: string,
65-
env: AuthenticatedEnvironment,
66-
showSecretDetails: boolean
65+
env: AuthenticatedEnvironment
6766
): Promise<RetrieveRunResponse | undefined> {
6867
return this.traceWithEnv("call", env, async (span) => {
6968
const taskRun = await this._replica.taskRun.findFirst({
@@ -72,11 +71,7 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
7271
runtimeEnvironmentId: env.id,
7372
},
7473
include: {
75-
attempts: {
76-
orderBy: {
77-
createdAt: "desc",
78-
},
79-
},
74+
attempts: true,
8075
lockedToVersion: true,
8176
schedule: true,
8277
tags: true,
@@ -111,50 +106,48 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
111106
let $output: any;
112107
let $outputPresignedUrl: string | undefined;
113108

114-
if (showSecretDetails) {
115-
const payloadPacket = await conditionallyImportPacket({
116-
data: taskRun.payload,
117-
dataType: taskRun.payloadType,
118-
});
109+
const payloadPacket = await conditionallyImportPacket({
110+
data: taskRun.payload,
111+
dataType: taskRun.payloadType,
112+
});
119113

120-
if (
121-
payloadPacket.dataType === "application/store" &&
122-
typeof payloadPacket.data === "string"
123-
) {
124-
$payloadPresignedUrl = await generatePresignedUrl(
125-
env.project.externalRef,
126-
env.slug,
127-
payloadPacket.data,
128-
"GET"
129-
);
130-
} else {
131-
$payload = await parsePacket(payloadPacket);
132-
}
114+
if (
115+
payloadPacket.dataType === "application/store" &&
116+
typeof payloadPacket.data === "string"
117+
) {
118+
$payloadPresignedUrl = await generatePresignedUrl(
119+
env.project.externalRef,
120+
env.slug,
121+
payloadPacket.data,
122+
"GET"
123+
);
124+
} else {
125+
$payload = await parsePacket(payloadPacket);
126+
}
133127

134-
if (taskRun.status === "COMPLETED_SUCCESSFULLY") {
135-
const completedAttempt = taskRun.attempts.find(
136-
(a) => a.status === "COMPLETED" && typeof a.output !== null
137-
);
128+
if (taskRun.status === "COMPLETED_SUCCESSFULLY") {
129+
const completedAttempt = taskRun.attempts.find(
130+
(a) => a.status === "COMPLETED" && typeof a.output !== null
131+
);
138132

139-
if (completedAttempt && completedAttempt.output) {
140-
const outputPacket = await conditionallyImportPacket({
141-
data: completedAttempt.output,
142-
dataType: completedAttempt.outputType,
143-
});
133+
if (completedAttempt && completedAttempt.output) {
134+
const outputPacket = await conditionallyImportPacket({
135+
data: completedAttempt.output,
136+
dataType: completedAttempt.outputType,
137+
});
144138

145-
if (
146-
outputPacket.dataType === "application/store" &&
147-
typeof outputPacket.data === "string"
148-
) {
149-
$outputPresignedUrl = await generatePresignedUrl(
150-
env.project.externalRef,
151-
env.slug,
152-
outputPacket.data,
153-
"GET"
154-
);
155-
} else {
156-
$output = await parsePacket(outputPacket);
157-
}
139+
if (
140+
outputPacket.dataType === "application/store" &&
141+
typeof outputPacket.data === "string"
142+
) {
143+
$outputPresignedUrl = await generatePresignedUrl(
144+
env.project.externalRef,
145+
env.slug,
146+
outputPacket.data,
147+
"GET"
148+
);
149+
} else {
150+
$output = await parsePacket(outputPacket);
158151
}
159152
}
160153
}
@@ -165,6 +158,7 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
165158
payloadPresignedUrl: $payloadPresignedUrl,
166159
output: $output,
167160
outputPresignedUrl: $outputPresignedUrl,
161+
error: ApiRetrieveRunPresenter.apiErrorFromError(taskRun.error),
168162
schedule: taskRun.schedule
169163
? {
170164
id: taskRun.schedule.friendlyId,
@@ -179,17 +173,9 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
179173
},
180174
}
181175
: undefined,
182-
attempts: !showSecretDetails
183-
? []
184-
: taskRun.attempts.map((a) => ({
185-
id: a.friendlyId,
186-
status: ApiRetrieveRunPresenter.apiStatusFromAttemptStatus(a.status),
187-
createdAt: a.createdAt ?? undefined,
188-
updatedAt: a.updatedAt ?? undefined,
189-
startedAt: a.startedAt ?? undefined,
190-
completedAt: a.completedAt ?? undefined,
191-
error: ApiRetrieveRunPresenter.apiErrorFromError(a.error),
192-
})),
176+
// We're removing attempts from the API
177+
attemptCount: taskRun.attempts.length,
178+
attempts: [],
193179
relatedRuns: {
194180
root: taskRun.rootTaskRun
195181
? await createCommonRunStructure(taskRun.rootTaskRun)

0 commit comments

Comments
 (0)