Skip to content

Commit

Permalink
Added support for another schema in IndexesHealth
Browse files Browse the repository at this point in the history
  • Loading branch information
ivan.vakhrushev committed Dec 22, 2019
1 parent a6d8f5f commit a24d9f5
Show file tree
Hide file tree
Showing 17 changed files with 479 additions and 340 deletions.
25 changes: 12 additions & 13 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,9 @@ You can call `pg_stat_reset()` to reset all statistics counters for the current

## Demo application
```java
import com.mfvanek.pg.connection.HighAvailabilityPgConnection;
import com.mfvanek.pg.connection.HighAvailabilityPgConnectionFactory;
import com.mfvanek.pg.connection.HighAvailabilityPgConnectionFactoryImpl;
import com.mfvanek.pg.connection.PgConnectionFactoryImpl;
import com.mfvanek.pg.index.health.logger.Exclusions;
import com.mfvanek.pg.index.health.logger.IndexesHealthLogger;
import com.mfvanek.pg.index.health.logger.SimpleHealthLogger;
import com.mfvanek.pg.index.maintenance.MaintenanceFactoryImpl;
import com.mfvanek.pg.model.MemoryUnit;
Expand All @@ -46,11 +43,12 @@ public class DemoApp {
final String readUrl = "jdbc:postgresql://host-name-1:6432,host-name-2:6432,host-name-3:6432/db_name_testing?targetServerType=preferSlave&loadBalanceHosts=true&ssl=true&prepareThreshold=0&preparedStatementCacheQueries=0&sslmode=require";
final String userName = "user_name_testing";
final String password = "password_testing";
final HighAvailabilityPgConnectionFactory haPgConnectionFactory = new HighAvailabilityPgConnectionFactoryImpl(new PgConnectionFactoryImpl());
final HighAvailabilityPgConnection haPgConnection = haPgConnectionFactory.of(writeUrl, userName, password, readUrl);
final IndexesHealth indexesHealth = new IndexesHealthImpl(haPgConnection, PgContext.ofPublic(), new MaintenanceFactoryImpl());
final IndexesHealthLogger logger = new SimpleHealthLogger(indexesHealth, Exclusions.empty());
logger.logAll().forEach(System.out::println);
final var haPgConnectionFactory = new HighAvailabilityPgConnectionFactoryImpl(new PgConnectionFactoryImpl());
final var haPgConnection = haPgConnectionFactory.of(writeUrl, userName, password, readUrl);
final var indexesHealth = new IndexesHealthImpl(haPgConnection, new MaintenanceFactoryImpl());
final var logger = new SimpleHealthLogger(indexesHealth, Exclusions.empty());
logger.logAll(PgContext.ofPublic())
.forEach(System.out::println);
// Resetting current statistics
// indexesHealth.resetStatistics();
}
Expand All @@ -61,15 +59,16 @@ public class DemoApp {
final String cascadeAsyncReadUrl = "jdbc:postgresql://host-name-6:6432/db_name_production?ssl=true&targetServerType=preferSlave&loadBalanceHosts=true&prepareThreshold=0&preparedStatementCacheQueries=0&connectTimeout=2&socketTimeout=50&loginTimeout=10&sslmode=require";
final String userName = "user_name_production";
final String password = "password_production";
final HighAvailabilityPgConnectionFactory haPgConnectionFactory = new HighAvailabilityPgConnectionFactoryImpl(new PgConnectionFactoryImpl());
final HighAvailabilityPgConnection haPgConnection = haPgConnectionFactory.of(writeUrl, userName, password, readUrl, cascadeAsyncReadUrl);
final IndexesHealth indexesHealth = new IndexesHealthImpl(haPgConnection, PgContext.ofPublic(), new MaintenanceFactoryImpl());
final var haPgConnectionFactory = new HighAvailabilityPgConnectionFactoryImpl(new PgConnectionFactoryImpl());
final var haPgConnection = haPgConnectionFactory.of(writeUrl, userName, password, readUrl, cascadeAsyncReadUrl);
final var indexesHealth = new IndexesHealthImpl(haPgConnection, new MaintenanceFactoryImpl());
final var exclusions = Exclusions.builder()
.withIndexSizeThreshold(10, MemoryUnit.MB)
.withTableSizeThreshold(10, MemoryUnit.MB)
.build();
final IndexesHealthLogger logger = new SimpleHealthLogger(indexesHealth, exclusions);
logger.logAll().forEach(System.out::println);
final var logger = new SimpleHealthLogger(indexesHealth, exclusions);
logger.logAll(PgContext.ofPublic())
.forEach(System.out::println);
}
}
```
57 changes: 49 additions & 8 deletions src/main/java/com/mfvanek/pg/index/health/IndexesHealth.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.mfvanek.pg.model.ForeignKey;
import com.mfvanek.pg.model.Index;
import com.mfvanek.pg.model.IndexWithNulls;
import com.mfvanek.pg.model.PgContext;
import com.mfvanek.pg.model.Table;
import com.mfvanek.pg.model.TableWithMissingIndex;
import com.mfvanek.pg.model.UnusedIndex;
Expand All @@ -19,31 +20,71 @@
public interface IndexesHealth {

@Nonnull
List<Index> getInvalidIndexes();
List<Index> getInvalidIndexes(@Nonnull PgContext pgContext);

@Nonnull
List<DuplicatedIndexes> getDuplicatedIndexes();
default List<Index> getInvalidIndexes() {
return getInvalidIndexes(PgContext.ofPublic());
}

@Nonnull
List<DuplicatedIndexes> getIntersectedIndexes();
List<DuplicatedIndexes> getDuplicatedIndexes(@Nonnull PgContext pgContext);

@Nonnull
List<UnusedIndex> getUnusedIndexes();
default List<DuplicatedIndexes> getDuplicatedIndexes() {
return getDuplicatedIndexes(PgContext.ofPublic());
}

@Nonnull
List<ForeignKey> getForeignKeysNotCoveredWithIndex();
List<DuplicatedIndexes> getIntersectedIndexes(@Nonnull PgContext pgContext);

@Nonnull
List<TableWithMissingIndex> getTablesWithMissingIndexes();
default List<DuplicatedIndexes> getIntersectedIndexes() {
return getIntersectedIndexes(PgContext.ofPublic());
}

@Nonnull
List<Table> getTablesWithoutPrimaryKey();
List<UnusedIndex> getUnusedIndexes(@Nonnull PgContext pgContext);

@Nonnull
default List<UnusedIndex> getUnusedIndexes() {
return getUnusedIndexes(PgContext.ofPublic());
}

@Nonnull
List<ForeignKey> getForeignKeysNotCoveredWithIndex(@Nonnull PgContext pgContext);

@Nonnull
default List<ForeignKey> getForeignKeysNotCoveredWithIndex() {
return getForeignKeysNotCoveredWithIndex(PgContext.ofPublic());
}

@Nonnull
List<TableWithMissingIndex> getTablesWithMissingIndexes(@Nonnull PgContext pgContext);

@Nonnull
default List<TableWithMissingIndex> getTablesWithMissingIndexes() {
return getTablesWithMissingIndexes(PgContext.ofPublic());
}

@Nonnull
List<Table> getTablesWithoutPrimaryKey(@Nonnull PgContext pgContext);

@Nonnull
default List<Table> getTablesWithoutPrimaryKey() {
return getTablesWithoutPrimaryKey(PgContext.ofPublic());
}

/**
* Get indexes that contain null values from all hosts.
*/
@Nonnull
List<IndexWithNulls> getIndexesWithNullValues();
List<IndexWithNulls> getIndexesWithNullValues(@Nonnull PgContext pgContext);

@Nonnull
default List<IndexWithNulls> getIndexesWithNullValues() {
return getIndexesWithNullValues(PgContext.ofPublic());
}

/**
* Reset all statistics counters on all hosts to zero.
Expand Down
39 changes: 20 additions & 19 deletions src/main/java/com/mfvanek/pg/index/health/IndexesHealthImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,79 +36,80 @@ public class IndexesHealthImpl implements IndexesHealth {
private final List<StatisticsMaintenance> statisticsMaintenanceForReplicas;

public IndexesHealthImpl(@Nonnull final HighAvailabilityPgConnection haPgConnection,
@Nonnull final PgContext pgContext,
@Nonnull final MaintenanceFactory maintenanceFactory) {
Objects.requireNonNull(haPgConnection);
Objects.requireNonNull(maintenanceFactory);
this.maintenanceForMaster = maintenanceFactory.forIndex(haPgConnection.getConnectionToMaster(), pgContext);
this.maintenanceForMaster = maintenanceFactory.forIndex(haPgConnection.getConnectionToMaster());
this.maintenanceForReplicas = ReplicasHelper.createIndexMaintenanceForReplicas(
haPgConnection.getConnectionsToReplicas(), pgContext, maintenanceFactory);
haPgConnection.getConnectionsToReplicas(), maintenanceFactory);
this.statisticsMaintenanceForReplicas = ReplicasHelper.createStatisticsMaintenanceForReplicas(
haPgConnection.getConnectionsToReplicas(), maintenanceFactory);
}

@Nonnull
@Override
public List<Index> getInvalidIndexes() {
public List<Index> getInvalidIndexes(@Nonnull final PgContext pgContext) {
logExecutingOnMaster();
return maintenanceForMaster.getInvalidIndexes();
return maintenanceForMaster.getInvalidIndexes(pgContext);
}

@Nonnull
@Override
public List<DuplicatedIndexes> getDuplicatedIndexes() {
public List<DuplicatedIndexes> getDuplicatedIndexes(@Nonnull final PgContext pgContext) {
logExecutingOnMaster();
return maintenanceForMaster.getDuplicatedIndexes();
return maintenanceForMaster.getDuplicatedIndexes(pgContext);
}

@Nonnull
@Override
public List<DuplicatedIndexes> getIntersectedIndexes() {
public List<DuplicatedIndexes> getIntersectedIndexes(@Nonnull final PgContext pgContext) {
logExecutingOnMaster();
return maintenanceForMaster.getIntersectedIndexes();
return maintenanceForMaster.getIntersectedIndexes(pgContext);
}

@Nonnull
@Override
public List<UnusedIndex> getUnusedIndexes() {
public List<UnusedIndex> getUnusedIndexes(@Nonnull final PgContext pgContext) {
final List<List<UnusedIndex>> potentiallyUnusedIndexesFromAllHosts = new ArrayList<>();
for (var maintenanceForReplica : maintenanceForReplicas) {
potentiallyUnusedIndexesFromAllHosts.add(
doOnHost(maintenanceForReplica.getHost(), maintenanceForReplica::getPotentiallyUnusedIndexes));
doOnHost(maintenanceForReplica.getHost(),
() -> maintenanceForReplica.getPotentiallyUnusedIndexes(pgContext)));
}
return ReplicasHelper.getUnusedIndexesAsIntersectionResult(potentiallyUnusedIndexesFromAllHosts);
}

@Nonnull
@Override
public List<ForeignKey> getForeignKeysNotCoveredWithIndex() {
public List<ForeignKey> getForeignKeysNotCoveredWithIndex(@Nonnull final PgContext pgContext) {
logExecutingOnMaster();
return maintenanceForMaster.getForeignKeysNotCoveredWithIndex();
return maintenanceForMaster.getForeignKeysNotCoveredWithIndex(pgContext);
}

@Nonnull
@Override
public List<TableWithMissingIndex> getTablesWithMissingIndexes() {
public List<TableWithMissingIndex> getTablesWithMissingIndexes(@Nonnull final PgContext pgContext) {
final List<List<TableWithMissingIndex>> tablesWithMissingIndexesFromAllHosts = new ArrayList<>();
for (var maintenanceForReplica : maintenanceForReplicas) {
tablesWithMissingIndexesFromAllHosts.add(
doOnHost(maintenanceForReplica.getHost(), maintenanceForReplica::getTablesWithMissingIndexes));
doOnHost(maintenanceForReplica.getHost(),
() -> maintenanceForReplica.getTablesWithMissingIndexes(pgContext)));
}
return ReplicasHelper.getTablesWithMissingIndexesAsUnionResult(tablesWithMissingIndexesFromAllHosts);
}

@Nonnull
@Override
public List<Table> getTablesWithoutPrimaryKey() {
public List<Table> getTablesWithoutPrimaryKey(@Nonnull final PgContext pgContext) {
logExecutingOnMaster();
return maintenanceForMaster.getTablesWithoutPrimaryKey();
return maintenanceForMaster.getTablesWithoutPrimaryKey(pgContext);
}

@Nonnull
@Override
public List<IndexWithNulls> getIndexesWithNullValues() {
public List<IndexWithNulls> getIndexesWithNullValues(@Nonnull final PgContext pgContext) {
logExecutingOnMaster();
return maintenanceForMaster.getIndexesWithNullValues();
return maintenanceForMaster.getIndexesWithNullValues(pgContext);
}

@Override
Expand Down
4 changes: 1 addition & 3 deletions src/main/java/com/mfvanek/pg/index/health/ReplicasHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import com.mfvanek.pg.index.maintenance.IndexMaintenance;
import com.mfvanek.pg.index.maintenance.MaintenanceFactory;
import com.mfvanek.pg.index.maintenance.StatisticsMaintenance;
import com.mfvanek.pg.model.PgContext;
import com.mfvanek.pg.model.TableWithMissingIndex;
import com.mfvanek.pg.model.UnusedIndex;
import org.apache.commons.collections4.CollectionUtils;
Expand All @@ -34,10 +33,9 @@ private ReplicasHelper() {
@Nonnull
static List<IndexMaintenance> createIndexMaintenanceForReplicas(
@Nonnull final Set<PgConnection> connectionsToReplicas,
@Nonnull final PgContext pgContext,
@Nonnull final MaintenanceFactory maintenanceFactory) {
return connectionsToReplicas.stream()
.map(con -> maintenanceFactory.forIndex(con, pgContext))
.map(maintenanceFactory::forIndex)
.collect(Collectors.toList());
}

Expand Down
51 changes: 26 additions & 25 deletions src/main/java/com/mfvanek/pg/index/health/logger/AbstractIndexesHealthLogger.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.mfvanek.pg.model.DuplicatedIndexes;
import com.mfvanek.pg.model.IndexNameAware;
import com.mfvanek.pg.model.IndexSizeAware;
import com.mfvanek.pg.model.PgContext;
import com.mfvanek.pg.model.TableNameAware;
import com.mfvanek.pg.model.TableSizeAware;
import org.apache.commons.collections4.CollectionUtils;
Expand Down Expand Up @@ -38,16 +39,16 @@ protected AbstractIndexesHealthLogger(@Nonnull final IndexesHealth indexesHealth

@Override
@Nonnull
public final List<String> logAll() {
public final List<String> logAll(@Nonnull final PgContext pgContext) {
final List<String> logResult = new ArrayList<>();
logResult.add(logInvalidIndexes());
logResult.add(logDuplicatedIndexes());
logResult.add(logIntersectedIndexes());
logResult.add(logUnusedIndexes());
logResult.add(logForeignKeysNotCoveredWithIndex());
logResult.add(logTablesWithMissingIndexes());
logResult.add(logTablesWithoutPrimaryKey());
logResult.add(logIndexesWithNullValues());
logResult.add(logInvalidIndexes(pgContext));
logResult.add(logDuplicatedIndexes(pgContext));
logResult.add(logIntersectedIndexes(pgContext));
logResult.add(logUnusedIndexes(pgContext));
logResult.add(logForeignKeysNotCoveredWithIndex(pgContext));
logResult.add(logTablesWithMissingIndexes(pgContext));
logResult.add(logTablesWithoutPrimaryKey(pgContext));
logResult.add(logIndexesWithNullValues(pgContext));
return logResult;
}

Expand All @@ -59,8 +60,8 @@ private String writeZeroToLog(@Nonnull final LoggingKey key) {
}

@Nonnull
private String logInvalidIndexes() {
final var invalidIndexes = indexesHealth.getInvalidIndexes();
private String logInvalidIndexes(@Nonnull final PgContext pgContext) {
final var invalidIndexes = indexesHealth.getInvalidIndexes(pgContext);
final LoggingKey key = SimpleLoggingKey.INVALID_INDEXES;
if (CollectionUtils.isNotEmpty(invalidIndexes)) {
LOGGER.error("There are invalid indexes in the database {}", invalidIndexes);
Expand All @@ -70,8 +71,8 @@ private String logInvalidIndexes() {
}

@Nonnull
private String logDuplicatedIndexes() {
final var rawDuplicatedIndexes = indexesHealth.getDuplicatedIndexes();
private String logDuplicatedIndexes(@Nonnull final PgContext pgContext) {
final var rawDuplicatedIndexes = indexesHealth.getDuplicatedIndexes(pgContext);
final var duplicatedIndexes = applyExclusions(rawDuplicatedIndexes,
exclusions.getDuplicatedIndexesExclusions());
final LoggingKey key = SimpleLoggingKey.DUPLICATED_INDEXES;
Expand All @@ -83,8 +84,8 @@ private String logDuplicatedIndexes() {
}

@Nonnull
private String logIntersectedIndexes() {
final var rawIntersectedIndexes = indexesHealth.getIntersectedIndexes();
private String logIntersectedIndexes(@Nonnull final PgContext pgContext) {
final var rawIntersectedIndexes = indexesHealth.getIntersectedIndexes(pgContext);
final var intersectedIndexes = applyExclusions(rawIntersectedIndexes,
exclusions.getIntersectedIndexesExclusions());
final LoggingKey key = SimpleLoggingKey.INTERSECTED_INDEXES;
Expand All @@ -96,8 +97,8 @@ private String logIntersectedIndexes() {
}

@Nonnull
private String logUnusedIndexes() {
final var rawUnusedIndexes = indexesHealth.getUnusedIndexes();
private String logUnusedIndexes(@Nonnull final PgContext pgContext) {
final var rawUnusedIndexes = indexesHealth.getUnusedIndexes(pgContext);
final var filteredUnusedIndexes = applyIndexesExclusions(
rawUnusedIndexes, exclusions.getUnusedIndexesExclusions());
final var unusedIndexes = applyIndexSizeExclusions(
Expand All @@ -111,8 +112,8 @@ private String logUnusedIndexes() {
}

@Nonnull
private String logForeignKeysNotCoveredWithIndex() {
final var foreignKeys = indexesHealth.getForeignKeysNotCoveredWithIndex();
private String logForeignKeysNotCoveredWithIndex(@Nonnull final PgContext pgContext) {
final var foreignKeys = indexesHealth.getForeignKeysNotCoveredWithIndex(pgContext);
final LoggingKey key = SimpleLoggingKey.FOREIGN_KEYS;
if (CollectionUtils.isNotEmpty(foreignKeys)) {
LOGGER.warn("There are foreign keys without index in the database {}", foreignKeys);
Expand All @@ -122,8 +123,8 @@ private String logForeignKeysNotCoveredWithIndex() {
}

@Nonnull
private String logTablesWithMissingIndexes() {
final var rawTablesWithMissingIndexes = indexesHealth.getTablesWithMissingIndexes();
private String logTablesWithMissingIndexes(@Nonnull final PgContext pgContext) {
final var rawTablesWithMissingIndexes = indexesHealth.getTablesWithMissingIndexes(pgContext);
final var tablesFilteredBySize = applyTableSizeExclusions(
rawTablesWithMissingIndexes, exclusions.getTableSizeThresholdInBytes());
final var tablesWithMissingIndexes = applyTablesExclusions(
Expand All @@ -137,8 +138,8 @@ private String logTablesWithMissingIndexes() {
}

@Nonnull
private String logTablesWithoutPrimaryKey() {
final var rawTablesWithoutPrimaryKey = indexesHealth.getTablesWithoutPrimaryKey();
private String logTablesWithoutPrimaryKey(@Nonnull final PgContext pgContext) {
final var rawTablesWithoutPrimaryKey = indexesHealth.getTablesWithoutPrimaryKey(pgContext);
final var tablesFilteredBySize = applyTableSizeExclusions(
rawTablesWithoutPrimaryKey, exclusions.getTableSizeThresholdInBytes());
final var tablesWithoutPrimaryKey = applyTablesExclusions(
Expand All @@ -152,8 +153,8 @@ private String logTablesWithoutPrimaryKey() {
}

@Nonnull
private String logIndexesWithNullValues() {
final var rawIndexesWithNullValues = indexesHealth.getIndexesWithNullValues();
private String logIndexesWithNullValues(@Nonnull final PgContext pgContext) {
final var rawIndexesWithNullValues = indexesHealth.getIndexesWithNullValues(pgContext);
final var indexesWithNullValues = applyIndexesExclusions(rawIndexesWithNullValues,
exclusions.getIndexesWithNullValuesExclusions());
final LoggingKey key = SimpleLoggingKey.INDEXES_WITH_NULLS;
Expand Down
Loading

0 comments on commit a24d9f5

Please sign in to comment.