Feature/protection against default config (#11)

* Added ServerSpecification model * Added PgParam * Added list of important params with default values * Base implementation for ConfigurationMaintenanceImpl * Refactoring * Added opportunity to filter out unused indexes by size * Corrected default values for important params * Updated copyright * Added possibility to get all params with their values
mfvanek · Dec 12, 2019 · dc5dd88 · dc5dd88
1 parent 65ba971
commit dc5dd88
Show file tree

Hide file tree

Showing 37 changed files with 809 additions and 85 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -26,6 +26,10 @@ ij_java_blank_lines_before_method_body = 0
 ij_java_blank_lines_before_package = 0
 ij_java_block_brace_style = end_of_line
 ij_java_case_statement_on_separate_line = true
+ij_java_keep_simple_blocks_in_one_line = true
+ij_java_keep_simple_classes_in_one_line = false
+ij_java_keep_simple_lambdas_in_one_line = true
+ij_java_keep_simple_methods_in_one_line = true
 ij_java_names_count_to_use_import_on_demand = 101
 ij_java_packages_to_use_import_on_demand = java.awt.*, javax.swing.*
 ij_java_prefer_longer_names = true

diff --git a/build.gradle b/build.gradle
@@ -3,7 +3,7 @@ plugins {
 }
 
 group 'com.mfvanek'
-version '0.0.1'
+version '0.0.2'
 
 sourceCompatibility = 11
 

diff --git a/src/main/java/com/mfvanek/pg/index/health/IndexesHealthImpl.java b/src/main/java/com/mfvanek/pg/index/health/IndexesHealthImpl.java
@@ -68,7 +68,7 @@ public List<UnusedIndex> getUnusedIndexes() {
         final List<List<UnusedIndex>> potentiallyUnusedIndexesFromAllHosts = new ArrayList<>();
         for (var maintenanceForReplica : maintenanceForReplicas) {
             potentiallyUnusedIndexesFromAllHosts.add(
-                    doOnReplica(maintenanceForReplica.getHost(), maintenanceForReplica::getPotentiallyUnusedIndexes));
+                    doOnHost(maintenanceForReplica.getHost(), maintenanceForReplica::getPotentiallyUnusedIndexes));
         }
         return ReplicasHelper.getUnusedIndexesAsIntersectionResult(potentiallyUnusedIndexesFromAllHosts);
     }
@@ -86,7 +86,7 @@ public List<TableWithMissingIndex> getTablesWithMissingIndexes() {
         final List<List<TableWithMissingIndex>> tablesWithMissingIndexesFromAllHosts = new ArrayList<>();
         for (var maintenanceForReplica : maintenanceForReplicas) {
             tablesWithMissingIndexesFromAllHosts.add(
-                    doOnReplica(maintenanceForReplica.getHost(), maintenanceForReplica::getTablesWithMissingIndexes));
+                    doOnHost(maintenanceForReplica.getHost(), maintenanceForReplica::getTablesWithMissingIndexes));
         }
         return ReplicasHelper.getTablesWithMissingIndexesAsUnionResult(tablesWithMissingIndexesFromAllHosts);
     }
@@ -109,8 +109,8 @@ private void logExecutingOnMaster() {
         LOGGER.debug("Going to execute on master host [{}]", maintenanceForMaster.getHost().getName());
     }
 
-    private <T> T doOnReplica(@Nonnull final PgHost host, Supplier<T> action) {
-        LOGGER.debug("Going to execute on replica host [{}]", host.getName());
+    private <T> T doOnHost(@Nonnull final PgHost host, Supplier<T> action) {
+        LOGGER.debug("Going to execute on host {}", host.getName());
         return action.get();
     }
 }
diff --git a/src/main/java/com/mfvanek/pg/index/health/logger/AbstractIndexesHealthLogger.java b/src/main/java/com/mfvanek/pg/index/health/logger/AbstractIndexesHealthLogger.java
@@ -7,8 +7,9 @@
 
 import com.mfvanek.pg.index.health.IndexesHealth;
 import com.mfvanek.pg.model.DuplicatedIndexes;
-import com.mfvanek.pg.model.IndexAware;
-import com.mfvanek.pg.model.TableAware;
+import com.mfvanek.pg.model.IndexNameAware;
+import com.mfvanek.pg.model.IndexSizeAware;
+import com.mfvanek.pg.model.TableNameAware;
 import org.apache.commons.collections4.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -95,7 +96,8 @@ private String logIntersectedIndexes() {
     @Nonnull
     private String logUnusedIndexes() {
         final var rawUnusedIndexes = indexesHealth.getUnusedIndexes();
-        final var unusedIndexes = applyIndexesExclusions(rawUnusedIndexes, exclusions.getUnusedIndexesExclusions());
+        final var filteredUnusedIndexes = applyIndexesExclusions(rawUnusedIndexes, exclusions.getUnusedIndexesExclusions());
+        final var unusedIndexes = applyIndexSizeExclusions(filteredUnusedIndexes, exclusions.getIndexSizeThreshold());
         final LoggingKey key = SimpleLoggingKey.UNUSED_INDEXES;
         if (CollectionUtils.isNotEmpty(unusedIndexes)) {
             LOGGER.warn("There are unused indexes in the database {}", unusedIndexes);
@@ -170,10 +172,9 @@ private List<DuplicatedIndexes> applyExclusions(@Nonnull final List<DuplicatedIn
     }
 
     @Nonnull
-    private static <T extends IndexAware> List<T> applyIndexesExclusions(@Nonnull final List<T> rawRecords,
-                                                                         @Nonnull final Set<String> exclusions) {
-        if (CollectionUtils.isEmpty(rawRecords) ||
-                CollectionUtils.isEmpty(exclusions)) {
+    private static <T extends IndexNameAware> List<T> applyIndexesExclusions(@Nonnull final List<T> rawRecords,
+                                                                             @Nonnull final Set<String> exclusions) {
+        if (CollectionUtils.isEmpty(rawRecords) || CollectionUtils.isEmpty(exclusions)) {
             return rawRecords;
         }
 
@@ -183,10 +184,21 @@ private static <T extends IndexAware> List<T> applyIndexesExclusions(@Nonnull fi
     }
 
     @Nonnull
-    private static <T extends TableAware> List<T> applyTablesExclusions(@Nonnull final List<T> rawRecords,
-                                                                        @Nonnull final Set<String> exclusions) {
-        if (CollectionUtils.isEmpty(rawRecords) ||
-                CollectionUtils.isEmpty(exclusions)) {
+    private static <T extends IndexSizeAware> List<T> applyIndexSizeExclusions(@Nonnull final List<T> rawRecords,
+                                                                               final long threshold) {
+        if (CollectionUtils.isEmpty(rawRecords) || threshold <= 0L) {
+            return rawRecords;
+        }
+
+        return rawRecords.stream()
+                .filter(i -> i.getIndexSizeInBytes() >= threshold)
+                .collect(Collectors.toList());
+    }
+
+    @Nonnull
+    private static <T extends TableNameAware> List<T> applyTablesExclusions(@Nonnull final List<T> rawRecords,
+                                                                            @Nonnull final Set<String> exclusions) {
+        if (CollectionUtils.isEmpty(rawRecords) || CollectionUtils.isEmpty(exclusions)) {
             return rawRecords;
         }
 

diff --git a/src/main/java/com/mfvanek/pg/index/health/logger/Exclusions.java b/src/main/java/com/mfvanek/pg/index/health/logger/Exclusions.java
@@ -5,6 +5,8 @@
 
 package com.mfvanek.pg.index.health.logger;
 
+import com.mfvanek.pg.model.MemoryUnit;
+import com.mfvanek.pg.utils.Validators;
 import org.apache.commons.lang3.StringUtils;
 
 import javax.annotation.Nonnull;
@@ -20,19 +22,22 @@ public class Exclusions {
     private final Set<String> tablesWithMissingIndexesExclusions;
     private final Set<String> tablesWithoutPrimaryKeyExclusions;
     private final Set<String> indexesWithNullValuesExclusions;
+    private final long indexSizeThreshold;
 
     private Exclusions(@Nonnull String duplicatedIndexesExclusions,
                        @Nonnull String intersectedIndexesExclusions,
                        @Nonnull String unusedIndexesExclusions,
                        @Nonnull String tablesWithMissingIndexesExclusions,
                        @Nonnull String tablesWithoutPrimaryKeyExclusions,
-                       @Nonnull String indexesWithNullValuesExclusions) {
+                       @Nonnull String indexesWithNullValuesExclusions,
+                       final long indexSizeThreshold) {
         this.duplicatedIndexesExclusions = prepareExclusions(duplicatedIndexesExclusions);
         this.intersectedIndexesExclusions = prepareExclusions(intersectedIndexesExclusions);
         this.unusedIndexesExclusions = prepareExclusions(unusedIndexesExclusions);
         this.tablesWithMissingIndexesExclusions = prepareExclusions(tablesWithMissingIndexesExclusions);
         this.tablesWithoutPrimaryKeyExclusions = prepareExclusions(tablesWithoutPrimaryKeyExclusions);
         this.indexesWithNullValuesExclusions = prepareExclusions(indexesWithNullValuesExclusions);
+        this.indexSizeThreshold = Validators.sizeNotNegative(indexSizeThreshold, "indexSizeThreshold");
     }
 
     private static Set<String> prepareExclusions(@Nonnull final String rawExclusions) {
@@ -79,6 +84,10 @@ public Set<String> getIndexesWithNullValuesExclusions() {
         return indexesWithNullValuesExclusions;
     }
 
+    public long getIndexSizeThreshold() {
+        return indexSizeThreshold;
+    }
+
     @Override
     public String toString() {
         return Exclusions.class.getSimpleName() + '{' +
@@ -88,6 +97,7 @@ public String toString() {
                 ", tablesWithMissingIndexesExclusions=" + tablesWithMissingIndexesExclusions +
                 ", tablesWithoutPrimaryKeyExclusions=" + tablesWithoutPrimaryKeyExclusions +
                 ", indexesWithNullValuesExclusions=" + indexesWithNullValuesExclusions +
+                ", indexSizeThreshold=" + indexSizeThreshold +
                 '}';
     }
 
@@ -109,6 +119,7 @@ public static class Builder {
         private String tablesWithMissingIndexesExclusions = EMPTY;
         private String tablesWithoutPrimaryKeyExclusions = EMPTY;
         private String indexesWithNullValuesExclusions = EMPTY;
+        private long indexSizeThreshold = 0L;
 
         private Builder() {
         }
@@ -143,14 +154,26 @@ public Builder withIndexesWithNullValuesExclusions(@Nonnull final String indexes
             return this;
         }
 
+        public Builder withIndexSizeThreshold(final long indexSizeThreshold) {
+            this.indexSizeThreshold = Validators.valueIsPositive(indexSizeThreshold, "indexSizeThreshold");
+            return this;
+        }
+
+        public Builder withIndexSizeThreshold(final int thresholdUnitsCount, final MemoryUnit unit) {
+            Validators.valueIsPositive(thresholdUnitsCount, "thresholdUnitsCount");
+            this.indexSizeThreshold = unit.convertToBytes(thresholdUnitsCount);
+            return this;
+        }
+
         public Exclusions build() {
             return new Exclusions(
                     duplicatedIndexesExclusions,
                     intersectedIndexesExclusions,
                     unusedIndexesExclusions,
                     tablesWithMissingIndexesExclusions,
                     tablesWithoutPrimaryKeyExclusions,
-                    indexesWithNullValuesExclusions);
+                    indexesWithNullValuesExclusions,
+                    indexSizeThreshold);
         }
 
         @Override
@@ -162,6 +185,7 @@ public String toString() {
                     ", tablesWithMissingIndexesExclusions='" + tablesWithMissingIndexesExclusions + '\'' +
                     ", tablesWithoutPrimaryKeyExclusions='" + tablesWithoutPrimaryKeyExclusions + '\'' +
                     ", indexesWithNullValuesExclusions='" + indexesWithNullValuesExclusions + '\'' +
+                    ", indexSizeThreshold=" + indexSizeThreshold +
                     '}';
         }
     }

diff --git a/src/main/java/com/mfvanek/pg/index/maintenance/ConfigurationMaintenance.java b/src/main/java/com/mfvanek/pg/index/maintenance/ConfigurationMaintenance.java
diff --git a/src/main/java/com/mfvanek/pg/index/maintenance/IndexMaintenanceImpl.java b/src/main/java/com/mfvanek/pg/index/maintenance/IndexMaintenanceImpl.java
@@ -14,23 +14,16 @@
 import com.mfvanek.pg.model.TableWithMissingIndex;
 import com.mfvanek.pg.model.TableWithoutPrimaryKey;
 import com.mfvanek.pg.model.UnusedIndex;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import com.mfvanek.pg.utils.QueryExecutor;
+import com.mfvanek.pg.utils.ResultSetExtractor;
 
 import javax.annotation.Nonnull;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 
 public class IndexMaintenanceImpl implements IndexMaintenance {
 
-    private static final Logger LOGGER = LoggerFactory.getLogger(IndexMaintenanceImpl.class);
-
     private static final String INVALID_INDEXES_SQL =
             "select x.indrelid::regclass as table_name, x.indexrelid::regclass as index_name\n" +
                     "from pg_index x\n" +
@@ -81,9 +74,7 @@ public class IndexMaintenanceImpl implements IndexMaintenance {
                     "where\n" +
                     "      psui.schemaname = 'public'::text and not i.indisunique and\n" +
                     "      i.indexrelid not in (select * from foreign_key_indexes) and /*retain indexes on foreign keys*/\n" +
-                    "      psui.idx_scan < 50::integer and\n" +
-                    "      pg_relation_size(psui.relid) >= 5::integer * 8192 and /*skip small tables*/\n" +
-                    "      pg_relation_size(psui.indexrelid) >= 5::integer * 8192 /*skip small indexes*/\n" +
+                    "      psui.idx_scan < 50::integer\n" +
                     "order by psui.relname, pg_relation_size(i.indexrelid) desc;";
 
     private static final String FOREIGN_KEYS_WITHOUT_INDEX =
@@ -254,21 +245,8 @@ public PgHost getHost() {
         return pgConnection.getHost();
     }
 
-    private <T> List<T> executeQuery(@Nonnull final String sqlQuery, ResultSetExtractor<T> rse) {
-        LOGGER.debug("Executing query: {}", sqlQuery);
-        try (Connection connection = pgConnection.getDataSource().getConnection();
-             Statement statement = connection.createStatement()) {
-            final List<T> executionResult = new ArrayList<>();
-            try (ResultSet resultSet = statement.executeQuery(Objects.requireNonNull(sqlQuery))) {
-                while (resultSet.next()) {
-                    executionResult.add(rse.extractData(resultSet));
-                }
-            }
-            LOGGER.debug("Query completed with result {}", executionResult);
-            return executionResult;
-        } catch (SQLException e) {
-            LOGGER.trace("Query failed", e);
-            throw new RuntimeException(e);
-        }
+    private <T> List<T> executeQuery(@Nonnull final String sqlQuery,
+                                     @Nonnull final ResultSetExtractor<T> rse) {
+        return QueryExecutor.executeQuery(pgConnection, sqlQuery, rse);
     }
 }
diff --git a/src/main/java/com/mfvanek/pg/model/DuplicatedIndexes.java b/src/main/java/com/mfvanek/pg/model/DuplicatedIndexes.java
@@ -5,6 +5,8 @@
 
 package com.mfvanek.pg.model;
 
+import com.mfvanek.pg.utils.Validators;
+
 import javax.annotation.Nonnull;
 import java.util.AbstractMap;
 import java.util.Arrays;
@@ -17,7 +19,7 @@
  * A typical error is when you create a column with an UNIQUE CONSTRAINT and then manually create an unique index on it.
  * See documentation https://www.postgresql.org/docs/10/ddl-constraints.html#DDL-CONSTRAINTS-UNIQUE-CONSTRAINTS.
  */
-public class DuplicatedIndexes implements TableAware {
+public class DuplicatedIndexes implements TableNameAware {
 
     private final List<IndexWithSize> duplicatedIndexes;
     private final long totalSize;

diff --git a/src/main/java/com/mfvanek/pg/model/ForeignKey.java b/src/main/java/com/mfvanek/pg/model/ForeignKey.java
@@ -5,10 +5,12 @@
 
 package com.mfvanek.pg.model;
 
+import com.mfvanek.pg.utils.Validators;
+
 import javax.annotation.Nonnull;
 import java.util.List;
 
-public class ForeignKey implements TableAware {
+public class ForeignKey implements TableNameAware {
 
     private final String tableName;
     private final String constraintName;

diff --git a/src/main/java/com/mfvanek/pg/model/Index.java b/src/main/java/com/mfvanek/pg/model/Index.java
@@ -5,10 +5,12 @@
 
 package com.mfvanek.pg.model;
 
+import com.mfvanek.pg.utils.Validators;
+
 import javax.annotation.Nonnull;
 import java.util.Objects;
 
-public class Index implements TableAware, IndexAware {
+public class Index implements TableNameAware, IndexNameAware {
 
     private final String tableName;
     private final String indexName;

diff --git a/...java/com/mfvanek/pg/model/IndexAware.java → .../com/mfvanek/pg/model/IndexNameAware.java b/...java/com/mfvanek/pg/model/IndexAware.java → .../com/mfvanek/pg/model/IndexNameAware.java
@@ -7,7 +7,7 @@
 
 import javax.annotation.Nonnull;
 
-public interface IndexAware {
+public interface IndexNameAware {
 
     @Nonnull
     String getIndexName();

diff --git a/.../java/com/mfvanek/pg/model/SizeAware.java → .../com/mfvanek/pg/model/IndexSizeAware.java b/.../java/com/mfvanek/pg/model/SizeAware.java → .../com/mfvanek/pg/model/IndexSizeAware.java
@@ -5,7 +5,7 @@
 
 package com.mfvanek.pg.model;
 
-public interface SizeAware {
+public interface IndexSizeAware {
 
     long getIndexSizeInBytes();
 }
diff --git a/src/main/java/com/mfvanek/pg/model/IndexWithNulls.java b/src/main/java/com/mfvanek/pg/model/IndexWithNulls.java
@@ -5,6 +5,8 @@
 
 package com.mfvanek.pg.model;
 
+import com.mfvanek.pg.utils.Validators;
+
 import javax.annotation.Nonnull;
 
 public final class IndexWithNulls extends IndexWithSize {

diff --git a/src/main/java/com/mfvanek/pg/model/IndexWithSize.java b/src/main/java/com/mfvanek/pg/model/IndexWithSize.java
@@ -5,9 +5,11 @@
 
 package com.mfvanek.pg.model;
 
+import com.mfvanek.pg.utils.Validators;
+
 import javax.annotation.Nonnull;
 
-public class IndexWithSize extends Index implements SizeAware {
+public class IndexWithSize extends Index implements IndexSizeAware {
 
     private final long indexSizeInBytes;