Feature/dmnd 1782 db config

api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt

@@ -9,8 +9,6 @@ import edu.wgu.osmt.richskill.RichSkillDoc
 import edu.wgu.osmt.richskill.RichSkillEsRepo
 import org.apache.lucene.search.join.ScoreMode
 import org.elasticsearch.index.query.*
-import org.elasticsearch.index.query.QueryBuilders.matchPhrasePrefixQuery
-import org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
@@ -93,17 +91,17 @@ class CustomCollectionQueriesImpl @Autowired constructor(
         publishStatus: Set<PublishStatus>,
         pageable: Pageable
     ): SearchHits<CollectionDoc> {
-        val nqb = NativeSearchQueryBuilder().withPageable(Pageable.unpaged())
+        val nsqb1 = NativeSearchQueryBuilder().withPageable(Pageable.unpaged())
         val bq = QueryBuilders.boolQuery()
-        //TODO Replace with FindsAllByPublishStatus.createTermsDslQuery(publishStatus.name, publishStatus.map { ps -> ps.toString() })
+        val filterDslQuery = createTermsDslQuery(RichSkillDoc::publishStatus.name, publishStatus.map { ps -> ps.toString() })
         val filter = BoolQueryBuilder().must(
             QueryBuilders.termsQuery(
                 RichSkillDoc::publishStatus.name,
                 publishStatus.map { ps -> ps.toString() }
             )
         )
-        nqb.withFilter(filter)
-        nqb.withQuery(bq)
+        nsqb1.withFilter(filter)
+        nsqb1.withQuery(bq)
 
         var collectionMultiPropertyResults: List<String> = listOf()
 
@@ -113,84 +111,57 @@ class CustomCollectionQueriesImpl @Autowired constructor(
             bq.should(
                 BoolQueryBuilder()
                     .must(richSkillEsRepo.richSkillPropertiesMultiMatch(apiSearch.query))
-                    .must(
-                        QueryBuilders.nestedQuery(
-                            RichSkillDoc::collections.name,
-                            QueryBuilders.matchAllQuery(),
-                            ScoreMode.Avg
-                        ).innerHit(InnerHitBuilder())
-                    )
+                    .must(createNestedQueryBuilder())
             )
             bq.should(richSkillEsRepo.occupationQueries(apiSearch.query))
 
-            val nqb = NativeSearchQueryBuilder()
+            val nsqb = NativeSearchQueryBuilder()
                 .withQuery( collectionPropertiesMultiMatch(apiSearch.query) )
                 .withPageable(Pageable.unpaged())
                 .withFilter(filter)
             // search on collection specific properties
-            val query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()1", nqb, log)
+            val query = convertToNativeQuery(Pageable.unpaged(), filterDslQuery, nsqb, "CustomCollectionQueriesImpl.byApiSearch()1", log)
             collectionMultiPropertyResults = elasticSearchTemplate
-                .search(query, CollectionDoc::class.java)
-                .searchHits
-                .map { it.content.uuid }
+                                                .search(query, CollectionDoc::class.java)
+                                                .searchHits
+                                                .map { it.content.uuid }
 
         } else if (apiSearch.advanced != null) {
             richSkillEsRepo.generateBoolQueriesFromApiSearch(bq, apiSearch.advanced)
 
             if (!apiSearch.advanced.collectionName.isNullOrBlank()) {
-                if (apiSearch.advanced.collectionName.contains("\"")) {
-                    val nqb = NativeSearchQueryBuilder()
-                        .withQuery( simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) )
-                        .withPageable(Pageable.unpaged())
-                        .withFilter(filter)
-                    val query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()2", nqb, log)
-                    collectionMultiPropertyResults = elasticSearchTemplate
-                        .search( query, CollectionDoc::class.java )
-                        .searchHits
-                        .map { it.content.uuid }
-                } else {
-                    val nqb = NativeSearchQueryBuilder()
-                        .withQuery( matchPhrasePrefixQuery( CollectionDoc::name.name, apiSearch.advanced.collectionName ) )
-                        .withPageable(Pageable.unpaged())
-                        .withFilter(filter)
-                    val query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()3", nqb, log)
-                    collectionMultiPropertyResults = elasticSearchTemplate
-                        .search( query, CollectionDoc::class.java )
-                        .searchHits
-                        .map { it.content.uuid }
-                }
+                collectionMultiPropertyResults = getCollectionUuids(pageable, filterDslQuery, apiSearch.advanced.collectionName )
             } else {
-                bq.must(
-                    QueryBuilders.nestedQuery(
-                        RichSkillDoc::collections.name,
-                        QueryBuilders.matchAllQuery(),
-                        ScoreMode.Avg
-                    ).innerHit(InnerHitBuilder())
-                )
+                bq.must(createNestedQueryBuilder())
             }
         } else { // query nor advanced search was provided, return all collections
-            bq.must(
-                QueryBuilders.nestedQuery(
-                    RichSkillDoc::collections.name,
-                    QueryBuilders.matchAllQuery(),
-                    ScoreMode.Avg
-                ).innerHit(InnerHitBuilder())
-            )
+            bq.must(createNestedQueryBuilder())
         }
 
-        var query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch().innerHitCollectionUuids", nqb, log)
-        val results = elasticSearchTemplate.search(query, RichSkillDoc::class.java)
+        var query = convertToNativeQuery(Pageable.unpaged(), filterDslQuery, nsqb1, "CustomCollectionQueriesImpl.byApiSearch().innerHitCollectionUuids", log)
+        val innerHitCollectionUuids = elasticSearchTemplate
+                                        .search(query, RichSkillDoc::class.java)
+                                        .searchHits.mapNotNull { it.getInnerHits("collections")?.searchHits?.mapNotNull { it.content as CollectionDoc } }
+                                        .flatten()
+                                        .map { it.uuid }
+                                        .distinct()
+        return getCollectionFromUuids(pageable, filterDslQuery, (innerHitCollectionUuids + collectionMultiPropertyResults).distinct())
+    }
 
-        val innerHitCollectionUuids =
-            results.searchHits.mapNotNull { it.getInnerHits("collections")?.searchHits?.mapNotNull { it.content as CollectionDoc } }
-                .flatten().map { it.uuid }.distinct()
+    @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createNestQueryDslQuery"), DeprecationLevel.WARNING )
+    private fun createNestedQueryBuilder(): NestedQueryBuilder {
+        return QueryBuilders.nestedQuery(
+                                            RichSkillDoc::collections.name,
+                                            QueryBuilders.matchAllQuery(),
+                                            ScoreMode.Avg
+                ).innerHit(InnerHitBuilder())
+    }
 
-        val nqb2 = NativeSearchQueryBuilder()
-            .withQuery( QueryBuilders.termsQuery( "_id", (innerHitCollectionUuids + collectionMultiPropertyResults).distinct() ) )
-            .withFilter(filter)
-            .withPageable(pageable)
-        query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()4", nqb2, log)
-        return elasticSearchTemplate.search(query, CollectionDoc::class.java)
+    private fun getCollectionUuids(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List<String> {
+        return  if (collectionName.contains("\""))
+                    getCollectionUuidsFromComplexName(pageable, filter, collectionName)
+                else
+                    getCollectionUuidsFromName(pageable, filter, collectionName)
     }
 }
 

api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt

@@ -43,28 +43,28 @@ fun SchemaUtils.addMissingColumnsStatementsPublic(vararg tables: Table): List<St
                 incorrectNullabilityColumns.flatMapTo(statements) { it.modifyStatement() }
             }
         }
-//TODO DMND-1782
-//        if (db.supportsAlterTableWithAddColumn) {
-//            val existingColumnConstraint = db.dialect.columnConstraints(*tables)
-//
-//            for (table in tables) {
-//                for (column in table.columns) {
-//                    val foreignKey = column.foreignKey
-//                    if (foreignKey != null) {
-//                        val existingConstraint = existingColumnConstraint[table to column]?.firstOrNull()
-//                        if (existingConstraint == null) {
-//                            statements.addAll(createFKey(column))
-//                        } else if (existingConstraint.target.table != foreignKey.target.table
-//                            || foreignKey.deleteRule != existingConstraint.deleteRule
-//                            || foreignKey.updateRule != existingConstraint.updateRule
-//                        ) {
-//                            statements.addAll(existingConstraint.dropStatement())
-//                            statements.addAll(createFKey(column))
-//                        }
-//                    }
-//                }
-//            }
-//        }
+
+        if (db.supportsAlterTableWithAddColumn) {
+            val existingColumnConstraint = db.dialect.columnConstraints(*tables)
+
+            for (table in tables) {
+                for (column in table.columns) {
+                    val foreignKey = column.foreignKey
+                    if (foreignKey != null) {
+                        val existingConstraint = existingColumnConstraint[table to column]?.firstOrNull()
+                        if (existingConstraint == null) {
+                            statements.addAll(createFKey(column))
+                        } else if (existingConstraint.target.table != foreignKey.target.table
+                            || foreignKey.deleteRule != existingConstraint.deleteRule
+                            || foreignKey.updateRule != existingConstraint.updateRule
+                        ) {
+                            statements.addAll(existingConstraint.dropStatement())
+                            statements.addAll(createFKey(column))
+                        }
+                    }
+                }
+            }
+        }
 
         return statements
     }

api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt

@@ -1,9 +1,12 @@
 package edu.wgu.osmt.elasticsearch
 
 import co.elastic.clients.elasticsearch._types.FieldValue
-import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders
+import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode
+import co.elastic.clients.elasticsearch._types.query_dsl.Operator
 import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.*
 import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField
+import co.elastic.clients.elasticsearch.core.search.InnerHits
+import edu.wgu.osmt.collection.CollectionDoc
 import edu.wgu.osmt.db.PublishStatus
 import edu.wgu.osmt.richskill.RichSkillDoc
 import org.slf4j.Logger
@@ -44,7 +47,7 @@ interface FindsAllByPublishStatus<T> {
                             .stream()
                             .map { ps -> ps.name}
                             .collect(Collectors.toList())
-        var filter = createTermsDslQuery(false, RichSkillDoc::publishStatus.name, filterValues)
+        var filter = createTermsDslQuery( RichSkillDoc::publishStatus.name, filterValues, false)
         return NativeQueryBuilder()
                     .withPageable(pageable)
                     .withQuery(MATCH_ALL)
@@ -59,9 +62,45 @@ interface FindsAllByPublishStatus<T> {
     }
 
     /**
-     * Create a query_dsl.Query instance that ElasticSearchTemplate v8.x mandates for filtering.
+     * Stepping stone to 100% migration to ES v8.7.x apis; see KeywordEsRepo.kt
      */
-    fun createTermsDslQuery(andFlag: Boolean, fieldName: String, filterValues: List<String>): co.elastic.clients.elasticsearch._types.query_dsl.Query {
+    fun convertToNativeQuery(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, nsqb: NativeSearchQueryBuilder, msgPrefix: String, log: Logger): Query {
+        val oldQuery = nsqb.build()
+        val nuQuery = NativeQuery.builder()
+            .withFilter(filter)
+            .withQuery(StringQuery(oldQuery.query.toString()))
+            .withPageable(pageable)
+            .build()
+        log.debug(String.Companion.format("\n%s springDataQuery:\n\t\t%s", msgPrefix, (nuQuery.springDataQuery as StringQuery).source))
+        log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, nuQuery.filter.toString()))
+        return nuQuery
+    }
+
+    /*
+     * Below methods are all leveraging the latest ElasticSearch v8.7.X Java API
+     */
+    fun createMatchPhrasePrefixDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query {
+        return matchPhrasePrefix { qb -> qb.field(fieldName).query(searchStr).boost(boostVal) }
+    }
+
+    fun createMatchBoolPrefixDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query {
+        return matchBoolPrefix { qb -> qb.field(fieldName).query(searchStr).boost(boostVal) }
+    }
+
+    fun createSimpleQueryDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query {
+        return simpleQueryString { qb -> qb.fields(fieldName).query(searchStr).boost(boostVal).defaultOperator(Operator.And) }
+    }
+
+    fun createNestedQueryDslQuery(path: String, scoreMode: ChildScoreMode, query: co.elastic.clients.elasticsearch._types.query_dsl.Query? = null, innerHits: InnerHits? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query {
+        query ?: matchAll { b-> b }
+        innerHits ?: InnerHits.Builder().build()
+        return nested { qb -> qb.path(path)
+                                .scoreMode(ChildScoreMode.Avg)
+                                .innerHits(innerHits)
+                                .query(matchAll { b-> b }) }
+    }
+
+    fun createTermsDslQuery(fieldName: String, filterValues: List<String>, andFlag: Boolean = true): co.elastic.clients.elasticsearch._types.query_dsl.Query {
         val values = filterValues
             .stream()
             .map { FieldValue.of(it) }
@@ -77,9 +116,9 @@ interface FindsAllByPublishStatus<T> {
         /* Short hand version https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html
         val terms2 = terms { qb -> qb.field(fieldName).terms(tqf) }
         return bool { qb -> if (andFlag)
-                                qb.must(terms)
+                                qb.must(terms2)
                             else
-                                qb.should(terms) }
+                                qb.should(terms2) }
         */
 
         return bool()
@@ -89,31 +128,39 @@ interface FindsAllByPublishStatus<T> {
             ._toQuery()
     }
 
-    fun createTermsDslQuery(fieldName: String, filterValues: List<String>): co.elastic.clients.elasticsearch._types.query_dsl.Query {
-        return createTermsDslQuery(true, fieldName, filterValues)
+    fun getCollectionUuidsFromComplexName(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List<String> {
+        val query = NativeQuery
+            .builder()
+            .withFilter(filter)
+            .withQuery(createSimpleQueryDslQuery("${CollectionDoc::name.name}.raw", collectionName))
+            .withPageable(pageable)
+            .build()
+        return elasticSearchTemplate
+            .search( query, CollectionDoc::class.java )
+            .searchHits
+            .map { it.content.uuid }
     }
 
-    @Deprecated("", ReplaceWith("convertToNativeQuery"), DeprecationLevel.WARNING)
-    fun convertToStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder, log: Logger): Query {
-        val query = nqb.build()
-        log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString()))
-        log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString()))
-        //NOTE: this causes us to lose the filter query
-        return StringQuery(query.query.toString())
+    fun getCollectionUuidsFromName(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List<String> {
+        val query = NativeQuery
+                            .builder()
+                            .withFilter(filter)
+                            .withQuery(createMatchPhrasePrefixDslQuery(CollectionDoc::name.name, collectionName))
+                            .withPageable(pageable)
+                            .build()
+        return elasticSearchTemplate
+                            .search( query, CollectionDoc::class.java )
+                            .searchHits
+                            .map { it.content.uuid }
     }
 
-    /**
-     * Stepping stone to 100% migration to ES v8.7.x apis; see KeywordEsRepo.kt
-     */
-    fun convertToNativeQuery(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, nsqb: NativeSearchQueryBuilder, msgPrefix: String, log: Logger): Query {
-        val oldQuery = nsqb.build()
-        val nuQuery = NativeQuery.builder()
+    fun getCollectionFromUuids(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, uuids: List<String> ): SearchHits<CollectionDoc> {
+        val query = NativeQuery
+            .builder()
             .withFilter(filter)
-            .withQuery(StringQuery(oldQuery.query.toString()))
+            .withQuery(createTermsDslQuery("_id", uuids))
             .withPageable(pageable)
             .build()
-        log.debug(String.Companion.format("\n%s springDataQuery:\n\t\t%s", msgPrefix, (nuQuery.springDataQuery as StringQuery).source))
-        log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, nuQuery.filter.toString()))
-        return nuQuery
+        return elasticSearchTemplate.search(query, CollectionDoc::class.java)
     }
 }

api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt

@@ -71,7 +71,7 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea
     }
 }
 
-@Deprecated("Upgrade to ES v8.x queries", ReplaceWith("Replacement method"), DeprecationLevel.WARNING )
+@Deprecated("Upgrade to ES v8.x queries", ReplaceWith("JobCodeQueriesEx"), DeprecationLevel.WARNING )
 object JobCodeQueries {
     //TODO Convert to ES v8.7.x apis and return the newer BoolQuery.Builder instance; see KeywordEsRep.kt
     fun multiPropertySearch(query: String, parentDocPath: String? = null): BoolQueryBuilder {