feat: improve event scraping

app/.env.example

@@ -1,14 +1,3 @@
-# Prisma
-# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
-DATABASE_URL="postgresql://postgres:admin@localhost:5432/db"
-
-# Next Auth
-# You can generate a new secret on the command line with:
-# openssl rand -base64 32
-# https://next-auth.js.org/configuration/options#secret
-NEXTAUTH_SECRET=""
-NEXTAUTH_URL="http://localhost"
-
 # Next Auth Discord Provider
 DISCORD_CLIENT_ID=""
 DISCORD_CLIENT_SECRET=""
@@ -33,7 +22,6 @@ UNLEASH_SERVER_API_TOKEN=""
 
 # Other
 NEXT_PUBLIC_CARE_BEAR_SHOOTER_BUILD_URL=""
-OPENAI_API_KEY=""
 
 # API
 AWS_ACCESS_KEY_ID="" # AWS_PROFILE=sinister-incorporated-test terraform output access_key_app_vercel

app/src/app/api/scrape-discord-events/route.ts

@@ -4,10 +4,12 @@ import { getEvents } from "@/discord/utils/getEvents";
 import { getEventUsers } from "@/discord/utils/getEventUsers";
 import type { eventSchema } from "@/discord/utils/schemas";
 import { env } from "@/env";
+import { log } from "@/logging";
 import { publishNotification } from "@/pusher/utils/publishNotification";
+import { getTracer } from "@/tracing/utils/getTracer";
+import { SpanStatusCode } from "@opentelemetry/api";
 import { NextResponse, type NextRequest } from "next/server";
 import { createHash } from "node:crypto";
-import { setTimeout } from "node:timers/promises";
 import { type z } from "zod";
 
 export async function POST(request: NextRequest) {
@@ -20,9 +22,10 @@ export async function POST(request: NextRequest) {
 
     const { data: events } = await getEvents();
 
+    void log.info("Scraping Discord events", {
+      events: events.map((event) => event.id),
+    });
     for (const event of events) {
-      await setTimeout(2000); // Cheap way to avoid rate limiting
-
       const hash = createHash("md5");
       hash.update(
         JSON.stringify({
@@ -87,59 +90,70 @@ export async function POST(request: NextRequest) {
 const updateParticipants = async (
   discordEvent: z.infer<typeof eventSchema>,
 ) => {
-  const databaseEvent = await prisma.discordEvent.findUnique({
-    where: {
-      discordId: discordEvent.id,
-    },
-  });
-  if (!databaseEvent) return;
-
-  const participants: { create: string[]; delete: string[] } = {
-    create: [],
-    delete: [],
-  };
-  const discordEventUserIds = (await getEventUsers(discordEvent.id)).map(
-    (user) => user.user_id,
-  );
-  const existingDatabaseParticipantIds = (
-    await prisma.discordEventParticipant.findMany({
-      where: {
-        event: {
+  return getTracer().startActiveSpan("updateParticipants", async (span) => {
+    try {
+      const databaseEvent = await prisma.discordEvent.findUnique({
+        where: {
           discordId: discordEvent.id,
         },
-      },
-    })
-  ).map((participant) => participant.discordUserId);
+      });
+      if (!databaseEvent) return;
 
-  // Collect new participants
-  for (const userId of discordEventUserIds) {
-    if (existingDatabaseParticipantIds.includes(userId)) continue;
-    participants.create.push(userId);
-  }
+      const participants: { create: string[]; delete: string[] } = {
+        create: [],
+        delete: [],
+      };
+      const discordEventUserIds = (await getEventUsers(discordEvent.id)).map(
+        (user) => user.user_id,
+      );
+      const existingDatabaseParticipantIds = (
+        await prisma.discordEventParticipant.findMany({
+          where: {
+            event: {
+              discordId: discordEvent.id,
+            },
+          },
+        })
+      ).map((participant) => participant.discordUserId);
 
-  // Collect removed participants
-  for (const userId of existingDatabaseParticipantIds) {
-    if (discordEventUserIds.includes(userId)) continue;
-    participants.delete.push(userId);
-  }
+      // Collect new participants
+      for (const userId of discordEventUserIds) {
+        if (existingDatabaseParticipantIds.includes(userId)) continue;
+        participants.create.push(userId);
+      }
 
-  // Save to database
-  if (participants.delete.length > 0) {
-    await prisma.discordEventParticipant.deleteMany({
-      where: {
-        eventId: databaseEvent.id,
-        discordUserId: {
-          in: participants.delete,
-        },
-      },
-    });
-  }
-  if (participants.create.length > 0) {
-    await prisma.discordEventParticipant.createMany({
-      data: participants.create.map((participantId) => ({
-        eventId: databaseEvent.id,
-        discordUserId: participantId,
-      })),
-    });
-  }
+      // Collect removed participants
+      for (const userId of existingDatabaseParticipantIds) {
+        if (discordEventUserIds.includes(userId)) continue;
+        participants.delete.push(userId);
+      }
+
+      // Save to database
+      if (participants.delete.length > 0) {
+        await prisma.discordEventParticipant.deleteMany({
+          where: {
+            eventId: databaseEvent.id,
+            discordUserId: {
+              in: participants.delete,
+            },
+          },
+        });
+      }
+      if (participants.create.length > 0) {
+        await prisma.discordEventParticipant.createMany({
+          data: participants.create.map((participantId) => ({
+            eventId: databaseEvent.id,
+            discordUserId: participantId,
+          })),
+        });
+      }
+    } catch (error) {
+      span.setStatus({
+        code: SpanStatusCode.ERROR,
+      });
+      throw error;
+    } finally {
+      span.end();
+    }
+  });
 };

app/src/discord/components/Event.tsx

@@ -4,18 +4,12 @@ import { Link } from "@/common/components/Link";
 import TimeAgoContainer from "@/common/components/TimeAgoContainer";
 import clsx from "clsx";
 import Image from "next/image";
+import type { z } from "zod";
+import type { eventSchema } from "../utils/schemas";
 
 type Props = Readonly<{
   className?: string;
-  event: {
-    id: string;
-    guild_id: string;
-    name: string;
-    image?: string | null;
-    scheduled_start_time: Date;
-    scheduled_end_time: Date;
-    user_count: number;
-  };
+  event: z.infer<typeof eventSchema>;
   index: number;
 }>;
 

app/src/discord/utils/getEvent.ts

@@ -4,7 +4,7 @@ import { SpanStatusCode } from "@opentelemetry/api";
 import { cache } from "react";
 import { z } from "zod";
 import { checkResponseForError } from "./checkResponseForError";
-import { userSchema } from "./schemas";
+import { eventSchema } from "./schemas";
 
 export const getEvent = cache(async (id: string) => {
   return getTracer().startActiveSpan("getEvent", async (span) => {
@@ -42,21 +42,7 @@ export const getEvent = cache(async (id: string) => {
   });
 });
 
-const successSchema = z.object({
-  id: z.string(),
-  guild_id: z.string(),
-  name: z.string(),
-  image: z.string().optional().nullable(),
-  scheduled_start_time: z.coerce.date(),
-  scheduled_end_time: z.coerce.date(),
-  user_count: z.number(),
-  description: z.string().optional(),
-  creator_id: z.string(),
-  creator: userSchema,
-  entity_metadata: z.object({
-    location: z.string().optional(),
-  }),
-});
+const successSchema = eventSchema;
 
 const errorSchema = z.object({
   message: z.string(),

app/src/discord/utils/getEventUsers.ts

@@ -1,6 +1,8 @@
 import { env } from "@/env";
+import { log } from "@/logging";
 import { getTracer } from "@/tracing/utils/getTracer";
 import { SpanStatusCode } from "@opentelemetry/api";
+import { setTimeout } from "node:timers/promises";
 import { cache } from "react";
 import { z } from "zod";
 import { checkResponseForError } from "./checkResponseForError";
@@ -9,18 +11,32 @@ import { memberSchema, userSchema } from "./schemas";
 export const getEventUsers = cache(async (id: string) => {
   return getTracer().startActiveSpan("getEventUsers", async (span) => {
     try {
-      // https://discord.com/developers/docs/resources/guild-scheduled-event#get-guild-scheduled-event-users
-      const response = await fetch(
-        `https://discord.com/api/v10/guilds/${env.DISCORD_GUILD_ID}/scheduled-events/${id}/users?with_member=true`,
-        {
-          headers: new Headers({
-            Authorization: `Bot ${env.DISCORD_TOKEN}`,
-          }),
-          next: {
-            revalidate: 30,
+      let response;
+      const maxRetries = 5;
+      for (let attempt = 0; attempt < maxRetries; attempt++) {
+        response = await fetch(
+          `https://discord.com/api/v10/guilds/${env.DISCORD_GUILD_ID}/scheduled-events/${id}/users?with_member=true`,
+          {
+            headers: new Headers({
+              Authorization: `Bot ${env.DISCORD_TOKEN}`,
+            }),
+            next: {
+              revalidate: 30,
+            },
           },
-        },
-      );
+        );
+        if (response.status !== 429) break;
+        const retryAfterHeader = response.headers.get("Retry-After");
+        const retryAfterSeconds = retryAfterHeader
+          ? Number.parseInt(retryAfterHeader, 10)
+          : 1;
+        void log.warn("Hit rate limit of Discord", {
+          endpoint: "getEventUsers",
+          retryAfter: retryAfterSeconds,
+        });
+        await setTimeout(retryAfterSeconds * 1000);
+      }
+      if (!response) throw new Error("Failed to fetch");
 
       const body: unknown = await response.json();
       const data = schema.parse(body);

app/src/discord/utils/schemas.ts

@@ -28,9 +28,11 @@ export const eventSchema = z.object({
   name: z.string(),
   image: z.string().optional().nullable(),
   scheduled_start_time: z.coerce.date(),
-  scheduled_end_time: z.coerce.date(),
+  scheduled_end_time: z.coerce.date().nullable(),
   user_count: z.number(),
   description: z.string().optional(),
+  creator_id: z.string(),
+  creator: userSchema,
   entity_metadata: z.object({
     location: z.string().optional(),
   }),

app/src/env.ts

@@ -7,17 +7,22 @@ export const env = createEnv({
    * Will throw if you access these variables on the client.
    */
   server: {
-    DATABASE_URL: z.string().url(),
+    DATABASE_URL: z
+      .string()
+      .url()
+      .default("postgresql://postgres:admin@localhost:5432/db"),
     NODE_ENV: z.enum(["development", "test", "production"]),
     NEXTAUTH_SECRET:
       process.env.NODE_ENV === "production"
         ? z.string().min(1)
         : z.string().min(1).optional(),
-    NEXTAUTH_URL: z.preprocess(
-      // Uses VERCEL_URL if NEXTAUTH_URL is not set, e.g. on Vercel's preview deployments
-      (str) => str || `https://${process.env.VERCEL_URL}`,
-      z.string().url(),
-    ),
+    NEXTAUTH_URL: z
+      .preprocess(
+        // Uses VERCEL_URL if NEXTAUTH_URL is not set, e.g. on Vercel's preview deployments
+        (str) => str || `https://${process.env.VERCEL_URL}`,
+        z.string().url(),
+      )
+      .default("http://localhost:3000"),
     DISCORD_CLIENT_ID: z.string(),
     DISCORD_CLIENT_SECRET: z.string(),
     DISCORD_GUILD_ID: z.string(),

app/src/events/components/OverviewTile.tsx

@@ -58,15 +58,15 @@ export const OverviewTile = ({ className, event, date }: Props) => {
 
             <dt className="text-neutral-500 mt-4">Ende</dt>
             <dd>
-              {event.scheduled_end_time.toLocaleString("de-DE", {
+              {event.scheduled_end_time?.toLocaleString("de-DE", {
                 timeZone: "Europe/Berlin",
                 weekday: "short",
                 year: "numeric",
                 month: "long",
                 day: "numeric",
                 hour: "2-digit",
                 minute: "2-digit",
-              })}
+              }) || "-"}
             </dd>
           </dl>
 

app/src/events/utils/getGoogleCalendarUrl.ts

@@ -8,7 +8,10 @@ export const getGoogleCalendarUrl = (
 
   const start = formatISO(event.scheduled_start_time, { format: "basic" });
 
-  const end = formatISO(event.scheduled_end_time, { format: "basic" });
+  const endDate = new Date(
+    event.scheduled_end_time || event.scheduled_start_time,
+  );
+  const end = formatISO(endDate, { format: "basic" });
 
   const description = event.description
     ? `&details=${encodeURIComponent(event.description)}`

app/src/events/utils/getIcsFile.ts

@@ -8,7 +8,10 @@ export const getIcsFile = (
   const start = format(event.scheduled_start_time, "yyyy-MM-dd-HH-mm")
     .split("-")
     .map(Number) as DateTime;
-  const end = format(event.scheduled_end_time, "yyyy-MM-dd-HH-mm")
+  const endDate = new Date(
+    event.scheduled_end_time || event.scheduled_start_time,
+  );
+  const end = format(endDate, "yyyy-MM-dd-HH-mm")
     .split("-")
     .map(Number) as DateTime;
 

app/src/events/utils/getOutlookUrl.ts

@@ -12,8 +12,11 @@ export const getOutlookUrl = (
     "yyyy-MM-dd'T'HH:mm:ss",
   );
 
+  const endDate = new Date(
+    event.scheduled_end_time || event.scheduled_start_time,
+  );
   const end = formatInTimeZone(
-    event.scheduled_end_time,
+    endDate,
     "Europe/Berlin",
     "yyyy-MM-dd'T'HH:mm:ss",
   );

bruno-collection/Discord/Event users.bru

@@ -5,7 +5,7 @@ meta {
 }
 
 get {
-  url: https://discord.com/api/v10/guilds/{{DISCORD_GUILD_ID}}/scheduled-events/1216706606486782033/users?with_member=true
+  url: https://discord.com/api/v10/guilds/{{DISCORD_GUILD_ID}}/scheduled-events/1310353937836150784/users?with_member=true
   body: none
   auth: none
 }

bruno-collection/Discord/Guild scheduled event.bru

@@ -5,12 +5,12 @@ meta {
 }
 
 get {
-  url: https://discord.com/api/v10/guilds/{{DISCORD_GUILD_ID}}/scheduled-events/1216706606486782033?with_user_count=true
+  url: https://discord.com/api/v10/guilds/{{DISCORD_GUILD_ID}}/scheduled-events/1334312197777788929?with_user_count=true
   body: none
   auth: none
 }
 
-query {
+params:query {
   with_user_count: true
 }