Geodatasets to use spatial index, also DB_POOL envs

NASA-AMMOS · Mar 13, 2024 · f5188db · f5188db
1 parent eddc4ae
commit f5188db
Show file tree

Hide file tree

Showing 5 changed files with 100 additions and 19 deletions.
diff --git a/API/Backend/Geodatasets/models/geodatasets.js b/API/Backend/Geodatasets/models/geodatasets.js
@@ -62,13 +62,32 @@ function makeNewGeodatasetTable(name, success, failure) {
           { where: { name: name }, silent: true }
         )
           .then((r) => {
-            success({
-              name: result.dataValues.name,
-              table: result.dataValues.table,
-              tableObj: GeodatasetTable,
-            });
+            sequelize
+              .query(
+                `CREATE INDEX IF NOT EXISTS ${result.dataValues.table}_geom_idx on ${result.dataValues.table} USING gist (geom);`
+              )
+              .then(() => {
+                success({
+                  name: result.dataValues.name,
+                  table: result.dataValues.table,
+                  tableObj: GeodatasetTable,
+                });
 
-            return null;
+                return null;
+              })
+              .catch((err) => {
+                logger(
+                  "error",
+                  "Failed to recreate spatial index for geodataset table.",
+                  "geodatasets",
+                  null,
+                  err
+                );
+                failure({
+                  status: "failure",
+                  message: "Failed to recreate spatial index",
+                });
+              });
           })
           .catch((err) => {
             logger(
@@ -89,6 +108,7 @@ function makeNewGeodatasetTable(name, success, failure) {
           .then(([results]) => {
             let newTable =
               "g" + (parseInt(results[0].count) + 1) + "_geodatasets";
+
             Geodatasets.create({
               name: name,
               table: newTable,
@@ -102,12 +122,31 @@ function makeNewGeodatasetTable(name, success, failure) {
                 sequelize
                   .sync()
                   .then(() => {
-                    success({
-                      name: name,
-                      table: newTable,
-                      tableObj: GeodatasetTable,
-                    });
-                    return null;
+                    sequelize
+                      .query(
+                        `CREATE INDEX ${newTable}_geom_idx on ${newTable} USING gist (geom);`
+                      )
+                      .then(() => {
+                        success({
+                          name: name,
+                          table: newTable,
+                          tableObj: GeodatasetTable,
+                        });
+                        return null;
+                      })
+                      .catch((err) => {
+                        logger(
+                          "error",
+                          "Failed to create spatial index for geodataset table.",
+                          "geodatasets",
+                          null,
+                          err
+                        );
+                        failure({
+                          status: "failure",
+                          message: "Failed to create spatial index",
+                        });
+                      });
                   })
                   .catch((err) => {
                     logger(

diff --git a/API/Backend/Geodatasets/routes/geodatasets.js b/API/Backend/Geodatasets/routes/geodatasets.js
@@ -461,15 +461,30 @@ function populateGeodatasetTable(Table, features, cb) {
 
   Table.bulkCreate(rows, { returning: true })
     .then(function (response) {
-      cb(true);
-      return null;
+      sequelize
+        .query(`VACUUM ANALYZE ${Table.tableName};`)
+        .then(() => {
+          cb(true);
+          return null;
+        })
+        .catch((err) => {
+          logger(
+            "error",
+            "Geodatasets: Failed to vacuum a geodataset spatial index!",
+            null,
+            null,
+            err
+          );
+          cb(false);
+          return null;
+        });
     })
     .catch(function (err) {
       logger(
         "error",
         "Geodatasets: Failed to populate a geodataset table!",
-        req.originalUrl,
-        req,
+        null,
+        null,
         err
       );
       cb(false);

diff --git a/API/connection.js b/API/connection.js
@@ -12,10 +12,19 @@ const sequelize = new Sequelize(
     dialect: "postgres",
     logging: process.env.VERBOSE_LOGGING == "true" || false,
     pool: {
-      max: 10,
+      max:
+        process.env.DB_POOL_MAX != null
+          ? parseInt(process.env.DB_POOL_MAX) || 10
+          : 10,
       min: 0,
-      acquire: 30000,
-      idle: 10000,
+      acquire:
+        process.env.DB_POOL_TIMEOUT != null
+          ? parseInt(process.env.DB_POOL_TIMEOUT) || 30000
+          : 30000,
+      idle:
+        process.env.DB_POOL_IDLE != null
+          ? parseInt(process.env.DB_POOL_IDLE) || 10000
+          : 10000,
     },
   }
 );

diff --git a/docs/pages/Setup/ENVs/ENVs.md b/docs/pages/Setup/ENVs/ENVs.md
@@ -58,6 +58,18 @@ User of Postgres database | string | default `null`
 
 Password of Postgres database | string | default `null`
 
+#### `DB_POOL_MAX=`
+
+Max number connections in the database's pool. CPUs \* 4 is a good number | integer | default `10`
+
+#### `DB_POOL_TIMEOUT=`
+
+How many milliseconds until a DB connection times out | integer | default `30000` (30 sec)
+
+#### `DB_POOL_IDLE=`
+
+How many milliseconds for an incoming connection to wait for a DB connection before getting kicked away | integer | default `10000` (10 sec)
+
 #### `CSSO_GROUPS=`
 
 A list of CSSO LDAP groups that have access | string[] | default `[]`

diff --git a/sample.env b/sample.env
@@ -49,6 +49,12 @@ DB_PORT=5432
 DB_NAME=name
 DB_USER=user
 DB_PASS=password
+# Max number connections in the database's pool. CPUs * 4 is a good number. Default is 10
+DB_POOL_MAX=
+# How many milliseconds until a DB connection times out. Default is 30000 (30 sec)
+DB_POOL_TIMEOUT=
+# How many milliseconds for an incoming connection to wait for a DB connection before getting kicked away. Default is 10000 (10 sec)
+DB_POOL_IDLE=
 
 #CONFIG
 # Disable the configure page