backup vacuum

app/config.server.ts

@@ -7,7 +7,6 @@ const loadEnv = () => {
   try {
     // Path to the .env file
     const envFilePath = path.resolve(__dirname, ".env");
-    console.log(envFilePath);
 
     // Read the contents of the .env file
     const envFileContent = fs.readFileSync(envFilePath, "utf8");
@@ -20,7 +19,6 @@ const loadEnv = () => {
       const [key, value] = line.split("=");
       if (key && value)
         process.env[key.trim()] = value.trim().replaceAll('"', "");
-      console.log(key, value);
     });
   } catch {
     return;
@@ -29,7 +27,6 @@ const loadEnv = () => {
 
 const config = () => {
   loadEnv();
-  console.log(process.env);
   const _config = z
     .object({
       ADMIN_EMAIL: z.string(),

app/gtfs/gtfs.ts

@@ -9,7 +9,6 @@ import { gtfsConfig, envConfig } from "../config.server";
 
 export const load = async () => {
   try {
-    console.log(envConfig);
     await importGtfs(gtfsConfig);
     await gtfsToHtml(gtfsConfig)
       .then(() => {

drizzle/backup.ts

@@ -1,7 +1,9 @@
-
+import fs from 'fs'
+import sqlite3 from 'sqlite3'
 import cron from 'node-cron';
 import { envConfig } from '~/config.server';
-import { S3Client, CopyObjectCommand } from '@aws-sdk/client-s3'
+import { S3Client, CopyObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3'
+import path from 'path';
 
 const s3Client = new S3Client({
   region: "us-east-1", // Replace with your AWS region
@@ -26,6 +28,86 @@ export async function backupS3Object(bucket: string, sourceKey: string, destinat
   }
 }
 
+const vacuumDb = (dbPath: string, backupPath: string) => {
+  // Open the SQLite database
+  const db = new sqlite3.Database(dbPath, sqlite3.OPEN_READWRITE, (err) => {
+    if (err) {
+      console.error('Error opening the database:', err.message);
+      return;
+    }
+
+    // Perform VACUUM operation
+    db.exec(`ATTACH DATABASE '${backupPath}' AS backup;`, (attachErr) => {
+      if (attachErr) {
+        console.error('Error attaching backup database:', attachErr.message);
+        db.close();
+        return;
+      }
+
+      db.exec('VACUUM INTO backup.main;', (vacuumErr) => {
+        if (vacuumErr) {
+          console.error('Error performing VACUUM:', vacuumErr.message);
+        } else {
+          console.log('VACUUM operation successful.');
+
+          // If needed, you can also copy the backup to another location
+          fs.copyFileSync(`${backupPath}.main`, `${backupPath}`);
+
+          console.log('Backup saved to:', backupPath);
+        }
+
+        // Detach the backup database
+        db.exec('DETACH DATABASE backup;', (detachErr) => {
+          if (detachErr) {
+            console.error('Error detaching backup database:', detachErr.message);
+          }
+
+          // Close the main database
+          db.close((closeErr) => {
+            if (closeErr) {
+              console.error('Error closing the database:', closeErr.message);
+            }
+          });
+        });
+      });
+    });
+  });
+}
+
+async function uploadFilesToS3(directory: string, prefix: string) {
+  // Read the contents of the local directory
+  const files = fs.readdirSync(directory);
+
+  // Upload each file to S3
+  for (const file of files) {
+    const filePath = path.join(directory, file);
+
+    // Skip directories
+    if (fs.statSync(filePath).isDirectory()) {
+      continue;
+    }
+
+    // Read the file content
+    const fileContent = fs.readFileSync(filePath);
+
+    // Create S3 PutObjectCommand with the specified prefix
+    const uploadParams = {
+      Bucket: envConfig.S3_BUCKET,
+      Key: `${prefix}/${file}`, // Include the prefix in the S3 key
+      Body: fileContent,
+    };
+
+    // Upload the file to S3
+    try {
+      const uploadResult = await s3Client.send(new PutObjectCommand(uploadParams));
+      console.log(`File uploaded successfully: ${file} - ETag: ${uploadResult.ETag}`);
+    } catch (error) {
+      console.error(`Error uploading file ${file} to S3:`, error);
+    }
+  }
+}
+
 cron.schedule('* * * * *', () => {
-  backupS3Object(envConfig.S3_BUCKET, 'data.db', '/drizzle/data.db')
+  vacuumDb('/drizzle/data.db', 'drizzle/backup')
+  uploadFilesToS3('drizzle/data.db', '/db-backup/data.db')
 })