OSMT-25: Export to Excel (.xlsx)

api/pom.xml

@@ -250,6 +250,16 @@
             <artifactId>opencsv</artifactId>
             <version>5.7.1</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi</artifactId>
+            <version>5.2.3</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi-ooxml</artifactId>
+            <version>5.2.3</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-actuator</artifactId>

api/src/main/kotlin/edu/wgu/osmt/ImportCommandRunner.kt

@@ -1,8 +1,8 @@
 package edu.wgu.osmt
 
-import edu.wgu.osmt.csv.BatchImportRichSkill
-import edu.wgu.osmt.csv.BlsImport
-import edu.wgu.osmt.csv.OnetImport
+import edu.wgu.osmt.io.csv.BatchImportRichSkill
+import edu.wgu.osmt.io.csv.BlsImport
+import edu.wgu.osmt.io.csv.OnetImport
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired

api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt

@@ -6,6 +6,8 @@ object RoutePaths {
     const val EXPORT = "$API/export"
     const val SEARCH_SKILLS = "$SEARCH_PATH/skills"
     const val EXPORT_LIBRARY = "$EXPORT/library"
+    const val EXPORT_LIBRARY_CSV = "$EXPORT_LIBRARY/csv"
+    const val EXPORT_LIBRARY_XLSX = "$EXPORT_LIBRARY/xlsx"
     const val SEARCH_SIMILAR_SKILLS = "$SEARCH_SKILLS/similarity"
     const val SEARCH_SIMILARITIES = "$SEARCH_SKILLS/similarities"
     const val SEARCH_COLLECTIONS = "$SEARCH_PATH/collections"
@@ -16,9 +18,12 @@ object RoutePaths {
     const val SKILLS_FILTER = "$SKILLS_PATH/filter"
     const val SKILL_PUBLISH = "$SKILLS_PATH/publish"
     const val SKILL_DETAIL = "$SKILLS_PATH/{uuid}"
+    const val SKILL_DETAIL_XLSX = "$SKILLS_PATH/{uuid}/xlsx"
     const val SKILL_UPDATE = "$SKILL_DETAIL/update"
     const val SKILL_AUDIT_LOG = "$SKILL_DETAIL/log"
     const val EXPORT_SKILLS = "$EXPORT/skills"
+    const val EXPORT_SKILLS_CSV = "$EXPORT_SKILLS/csv"
+    const val EXPORT_SKILLS_XLSX = "$EXPORT_SKILLS/xlsx"
 
 
     const val COLLECTIONS_PATH = "$API/collections"
@@ -31,12 +36,14 @@ object RoutePaths {
     const val COLLECTION_SKILLS = "$COLLECTION_DETAIL/skills"
     const val COLLECTION_AUDIT_LOG = "$COLLECTION_DETAIL/log"
     const val COLLECTION_CSV = "$COLLECTION_DETAIL/csv"
+    const val COLLECTION_XLSX = "$COLLECTION_DETAIL/xlsx"
     const val COLLECTION_REMOVE = "$COLLECTION_DETAIL/remove"
 
     const val WORKSPACE_PATH = "$API/workspace"
 
     const val TASKS_PATH = "$API/results"
     const val TASK_DETAIL_TEXT = "$TASKS_PATH/text/{uuid}"
+    const val TASK_DETAIL_MEDIA = "$TASKS_PATH/media/{uuid}"
     const val TASK_DETAIL_BATCH = "$TASKS_PATH/batch/{uuid}"
     const val TASK_DETAIL_SKILLS = "$TASKS_PATH/skills/{uuid}"
 

api/src/main/kotlin/edu/wgu/osmt/auditlog/AuditLogUtils.kt

@@ -4,7 +4,7 @@ import edu.wgu.osmt.collection.CollectionRepository
 import edu.wgu.osmt.collection.CollectionTable
 import edu.wgu.osmt.collection.diff
 import edu.wgu.osmt.config.AppConfig
-import edu.wgu.osmt.csv.BatchImportRichSkill
+import edu.wgu.osmt.io.csv.BatchImportRichSkill
 import edu.wgu.osmt.richskill.RichSkillDescriptorTable
 import edu.wgu.osmt.richskill.RichSkillRepository
 import edu.wgu.osmt.richskill.diff

api/src/main/kotlin/edu/wgu/osmt/collection/CollectionController.kt

@@ -26,6 +26,7 @@ import edu.wgu.osmt.task.Task
 import edu.wgu.osmt.task.TaskMessageService
 import edu.wgu.osmt.task.TaskResult
 import edu.wgu.osmt.task.UpdateCollectionSkillsTask
+import edu.wgu.osmt.task.XlsxTask
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.http.HttpEntity
 import org.springframework.http.HttpStatus
@@ -185,6 +186,18 @@ class CollectionController @Autowired constructor(
         return Task.processingResponse(task)
     }
 
+    @GetMapping(RoutePaths.COLLECTION_XLSX, produces = [MediaType.APPLICATION_OCTET_STREAM_VALUE])
+    fun getSkillsForCollectionXlsx(
+        @PathVariable uuid: String
+    ): HttpEntity<TaskResult> {
+        if (collectionRepository.findByUUID(uuid)!!.status == PublishStatus.Draft && !oAuthHelper.hasRole(appConfig.roleAdmin)) {
+            throw ResponseStatusException(HttpStatus.UNAUTHORIZED)
+        }
+        val task = XlsxTask(collectionUuid = uuid)
+        taskMessageService.enqueueJob(TaskMessageService.skillsForCollectionXlsx, task)
+        return Task.processingResponse(task)
+    }
+
     @DeleteMapping(RoutePaths.COLLECTION_REMOVE, produces = [MediaType.APPLICATION_JSON_VALUE])
     fun removeCollection(
         @PathVariable uuid: String

api/src/main/kotlin/edu/wgu/osmt/io/common/TabularResource.kt

@@ -0,0 +1,12 @@
+package edu.wgu.osmt.io.common
+
+interface TabularResource<S: TabColumn<T>, T> {
+
+    /**
+     * Defines the columns of this table in their desired order.
+     */
+    fun columnTranslations(data: List<T>): Array<S>
+
+}
+
+interface TabColumn<T>

api/src/main/kotlin/edu/wgu/osmt/io/common/TabularTask.kt

@@ -0,0 +1,26 @@
+package edu.wgu.osmt.io.common
+
+import edu.wgu.osmt.collection.CollectionRepository
+import edu.wgu.osmt.config.AppConfig
+import edu.wgu.osmt.richskill.RichSkillRepository
+import edu.wgu.osmt.task.Task
+import edu.wgu.osmt.task.TaskMessageService
+import org.springframework.beans.factory.annotation.Autowired
+
+abstract class TabularTask<T: Task> {
+    @Autowired
+    lateinit var taskMessageService: TaskMessageService
+
+    @Autowired
+    lateinit var collectionRepository: CollectionRepository
+
+    @Autowired
+    lateinit var richSkillRepository: RichSkillRepository
+
+    @Autowired
+    lateinit var appConfig: AppConfig
+
+    abstract fun tabularSkillsInCollectionProcessor(task: T)
+
+    abstract fun tabularSkillsInFullLibraryProcessor(task: T)
+}

api/src/main/kotlin/edu/wgu/osmt/csv/BatchImportRichSkill.kt → api/src/main/kotlin/edu/wgu/osmt/io/csv/BatchImportRichSkill.kt

@@ -1,4 +1,4 @@
-package edu.wgu.osmt.csv
+package edu.wgu.osmt.io.csv
 
 import com.opencsv.bean.CsvBindByName
 import edu.wgu.osmt.collection.CollectionDao

api/src/main/kotlin/edu/wgu/osmt/csv/BlsImport.kt → api/src/main/kotlin/edu/wgu/osmt/io/csv/BlsImport.kt

@@ -1,4 +1,4 @@
-package edu.wgu.osmt.csv
+package edu.wgu.osmt.io.csv
 
 import com.opencsv.bean.CsvBindByName
 import edu.wgu.osmt.config.AppConfig
@@ -10,7 +10,6 @@ import edu.wgu.osmt.richskill.RichSkillRepository
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
-import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
 /**

api/src/main/kotlin/edu/wgu/osmt/csv/CsvBuilder.kt → api/src/main/kotlin/edu/wgu/osmt/io/csv/CsvBuilder.kt

@@ -1,4 +1,4 @@
-package edu.wgu.osmt.csv
+package edu.wgu.osmt.io.csv
 
 /**
  * DSL style entry-point to build a CsvResource.  This approach should be used if a csv file is simple in structure
@@ -48,13 +48,13 @@ class CsvColumnBuilder<T> {
 }
 
 class CsvConfigBuilder<T> {
-    var delimeter: Char = CsvConfig.delimeter
+    var delimiter: Char = CsvConfig.delimiter
     var quoteChar: Char = CsvConfig.quoteChar
     var escapeChar: Char = CsvConfig.escapeChar
     var lineEnd: String = CsvConfig.lineEnd
     var includeHeader: Boolean = CsvConfig.includeHeader
 
     fun build(): CsvConfig {
-        return CsvConfig(delimeter, quoteChar, escapeChar, lineEnd, includeHeader)
+        return CsvConfig(delimiter, quoteChar, escapeChar, lineEnd, includeHeader)
     }
 }

api/src/main/kotlin/edu/wgu/osmt/csv/CsvImport.kt → api/src/main/kotlin/edu/wgu/osmt/io/csv/CsvImport.kt

@@ -1,4 +1,4 @@
-package edu.wgu.osmt.csv
+package edu.wgu.osmt.io.csv
 
 import com.opencsv.bean.CsvToBeanBuilder
 import edu.wgu.osmt.jobcode.JobCodeBreakout

api/src/main/kotlin/edu/wgu/osmt/csv/CsvResource.kt → api/src/main/kotlin/edu/wgu/osmt/io/csv/CsvResource.kt

@@ -1,15 +1,17 @@
-package edu.wgu.osmt.csv
+package edu.wgu.osmt.io.csv
 
 import com.opencsv.CSVWriter
+import edu.wgu.osmt.io.common.TabularResource
+import edu.wgu.osmt.io.common.TabColumn
 import java.io.StringWriter
 import java.io.Writer
 
-abstract class CsvResource<T>(val debugName: String) {
+abstract class CsvResource<T>(val debugName: String) : TabularResource<CsvColumn<T>, T> {
 
     /**
      * Defines the columns of this csv in their desired order.
      */
-    abstract fun columnTranslations(data: List<T>): Array<CsvColumn<T>>
+    abstract override fun columnTranslations(data: List<T>): Array<CsvColumn<T>>
 
     /**
      * Override if opencsv defaults are not desired
@@ -40,7 +42,7 @@ abstract class CsvResource<T>(val debugName: String) {
         val config = configureCsv()
 
         return CSVWriter(writer,
-                config.delimeter,
+                config.delimiter,
                 config.quoteChar,
                 config.escapeChar,
                 config.lineEnd
@@ -79,20 +81,20 @@ abstract class CsvResource<T>(val debugName: String) {
 data class CsvColumn<T>(
         val name: String = "",
         val translate: (T) -> String
-)
+): TabColumn<T>
 
 /**
  * Configure the global attributes of a csv export
  */
 data class CsvConfig(
-        val delimeter: Char = CsvConfig.delimeter,
-        val quoteChar: Char = CsvConfig.quoteChar,
-        val escapeChar: Char = CsvConfig.escapeChar,
-        val lineEnd: String = CsvConfig.lineEnd,
-        val includeHeader: Boolean = CsvConfig.includeHeader
+    val delimiter: Char = Defaults.delimiter,
+    val quoteChar: Char = Defaults.quoteChar,
+    val escapeChar: Char = Defaults.escapeChar,
+    val lineEnd: String = Defaults.lineEnd,
+    val includeHeader: Boolean = Defaults.includeHeader
 ) {
-    companion object Defaults { // Allows the default values to be shared with it's builder
-        val delimeter: Char = CSVWriter.DEFAULT_SEPARATOR
+    companion object Defaults { // Allows the default values to be shared with its builder
+        val delimiter: Char = CSVWriter.DEFAULT_SEPARATOR
         val quoteChar: Char = CSVWriter.DEFAULT_QUOTE_CHARACTER
         val escapeChar: Char = CSVWriter.DEFAULT_ESCAPE_CHARACTER
         val lineEnd: String = CSVWriter.DEFAULT_LINE_END

api/src/main/kotlin/edu/wgu/osmt/collection/CsvTaskProcessor.kt → api/src/main/kotlin/edu/wgu/osmt/io/csv/CsvTaskProcessor.kt

@@ -1,61 +1,45 @@
-package edu.wgu.osmt.collection
+package edu.wgu.osmt.io.csv
 
 import com.github.sonus21.rqueue.annotation.RqueueListener
-import edu.wgu.osmt.config.AppConfig
 import edu.wgu.osmt.db.PublishStatus
+import edu.wgu.osmt.io.common.TabularTask
 import edu.wgu.osmt.richskill.RichSkillAndCollections
-import edu.wgu.osmt.richskill.RichSkillCsvExport
 import edu.wgu.osmt.richskill.RichSkillDescriptorDao
-import edu.wgu.osmt.richskill.RichSkillRepository
 import edu.wgu.osmt.task.CsvTask
 import edu.wgu.osmt.task.TaskMessageService
 import edu.wgu.osmt.task.TaskStatus
 import org.jetbrains.exposed.dao.with
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
-import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.context.annotation.Profile
 import org.springframework.stereotype.Component
 import org.springframework.transaction.annotation.Transactional
 
 @Component
 @Profile("apiserver")
 @Transactional
-class CsvTaskProcessor {
+class CsvTaskProcessor : TabularTask<CsvTask>() {
     val logger: Logger = LoggerFactory.getLogger(CsvTaskProcessor::class.java)
 
-    @Autowired
-    lateinit var taskMessageService: TaskMessageService
-
-    @Autowired
-    lateinit var collectionRepository: CollectionRepository
-
-    @Autowired
-    lateinit var richSkillRepository: RichSkillRepository
-
-    @Autowired
-    lateinit var appConfig: AppConfig
-
     @RqueueListener(
         value = [TaskMessageService.skillsForCollectionCsv],
         deadLetterQueueListenerEnabled = "true",
         deadLetterQueue = TaskMessageService.deadLetters,
         concurrency = "1"
     )
-    fun csvSkillsInCollectionProcessor(csvTask: CsvTask) {
-        logger.info("Started processing task id: ${csvTask.uuid}")
+    override fun tabularSkillsInCollectionProcessor(task: CsvTask) {
+        logger.info("Started processing task id: ${task.uuid}")
 
-        val csv = collectionRepository.findByUUID(csvTask.collectionUuid)
+        val csv = collectionRepository.findByUUID(task.collectionUuid)
             ?.skills
             ?.filter { PublishStatus.Archived != it.publishStatus() }
-            ?.with(RichSkillDescriptorDao::collections)
             ?.map { RichSkillAndCollections.fromDao(it) }
             ?.let { RichSkillCsvExport(appConfig).toCsv(it) }
 
         taskMessageService.publishResult(
-            csvTask.copy(result = csv, status = TaskStatus.Ready)
+            task.copy(result = csv, status = TaskStatus.Ready)
         )
-        logger.info("Task ${csvTask.uuid} completed")
+        logger.info("Task ${task.uuid} completed")
     }
 
     @RqueueListener(
@@ -64,18 +48,17 @@ class CsvTaskProcessor {
         deadLetterQueue = TaskMessageService.deadLetters,
         concurrency = "1"
     )
-    fun csvSkillsInFullLibraryProcessor(csvTask: CsvTask) {
-        logger.info("Started processing task for Full Library export")
+    override fun tabularSkillsInFullLibraryProcessor(task: CsvTask) {
+        logger.info("Started processing task for Full Library .csv export")
 
         val csv = richSkillRepository.findAll()
-            ?.with(RichSkillDescriptorDao::collections)
             ?.map { RichSkillAndCollections.fromDao(it) }
             ?.let { RichSkillCsvExport(appConfig).toCsv(it) }
 
         taskMessageService.publishResult(
-            csvTask.copy(result = csv, status = TaskStatus.Ready)
+            task.copy(result = csv, status = TaskStatus.Ready)
         )
-        logger.info("Full Library export task completed")
+        logger.info("Full Library export task .csv completed")
     }
 
 }

api/src/main/kotlin/edu/wgu/osmt/richskill/ExportSkillToCsvTaskProcessor.kt → api/src/main/kotlin/edu/wgu/osmt/io/csv/ExportSkillToCsvTaskProcessor.kt

@@ -1,7 +1,9 @@
-package edu.wgu.osmt.richskill
+package edu.wgu.osmt.io.csv
 
 import com.github.sonus21.rqueue.annotation.RqueueListener
 import edu.wgu.osmt.config.AppConfig
+import edu.wgu.osmt.richskill.RichSkillAndCollections
+import edu.wgu.osmt.richskill.RichSkillRepository
 import edu.wgu.osmt.task.ExportSkillsToCsvTask
 import edu.wgu.osmt.task.TaskMessageService
 import edu.wgu.osmt.task.TaskStatus

api/src/main/kotlin/edu/wgu/osmt/csv/OnetImport.kt → api/src/main/kotlin/edu/wgu/osmt/io/csv/OnetImport.kt

@@ -1,4 +1,4 @@
-package edu.wgu.osmt.csv
+package edu.wgu.osmt.io.csv
 
 import com.opencsv.bean.CsvBindByName
 import edu.wgu.osmt.config.AppConfig

api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillCsvExport.kt → api/src/main/kotlin/edu/wgu/osmt/io/csv/RichSkillCsvExport.kt

@@ -1,10 +1,9 @@
-package edu.wgu.osmt.richskill
+package edu.wgu.osmt.io.csv
 
 import edu.wgu.osmt.config.AppConfig
-import edu.wgu.osmt.csv.CsvColumn
-import edu.wgu.osmt.csv.CsvResource
 import edu.wgu.osmt.jobcode.JobCode
 import edu.wgu.osmt.jobcode.JobCodeBreakout
+import edu.wgu.osmt.richskill.RichSkillAndCollections
 
 class RichSkillCsvExport(
     private val appConfig: AppConfig
@@ -15,18 +14,18 @@ class RichSkillCsvExport(
         val columns = arrayOf(
             CsvColumn<RichSkillAndCollections>("Canonical URL") { it.rs.canonicalUrl(appConfig.baseUrl) },
             CsvColumn("RSD Name") { it.rs.name },
-            CsvColumn("Authors") { it.rs.authors.map { author -> author.value ?: "" }.joinToString(listDelimiter) },
+            CsvColumn("Authors") { it.rs.authors.joinToString(listDelimiter) { author -> author.value ?: "" } },
             CsvColumn("Skill Statement") { it.rs.statement },
-            CsvColumn("Categories") { it.rs.categories.map{ category -> category.value ?: "" }.joinToString(listDelimiter) },
-            CsvColumn("Keywords") { it.rs.searchingKeywords.map { keyword -> keyword.value ?: "" }.joinToString(listDelimiter) },
-            CsvColumn("Standards") { it.rs.standards.map { keyword -> keyword.value ?: "" }.joinToString(listDelimiter) },
-            CsvColumn("Certifications") { it.rs.certifications.map { keyword -> keyword.value ?: "" }.joinToString(listDelimiter) },
+            CsvColumn("Categories") { it.rs.categories.joinToString(listDelimiter) { category -> category.value ?: "" } },
+            CsvColumn("Keywords") { it.rs.searchingKeywords.joinToString(listDelimiter) { keyword -> keyword.value ?: "" } },
+            CsvColumn("Standards") { it.rs.standards.joinToString(listDelimiter) { keyword -> keyword.value ?: "" } },
+            CsvColumn("Certifications") { it.rs.certifications.joinToString(listDelimiter) { keyword -> keyword.value ?: "" } },
             CsvColumn("Occupation Major Groups") { prepareJobCodePart(it.rs.jobCodes, JobCodeBreakout::majorCode) },
             CsvColumn("Occupation Minor Groups") { prepareJobCodePart(it.rs.jobCodes, JobCodeBreakout::minorCode) },
             CsvColumn("Broad Occupations") { prepareJobCodePart(it.rs.jobCodes, JobCodeBreakout::broadCode) },
             CsvColumn("Detailed Occupations") { prepareJobCodePart(it.rs.jobCodes, JobCodeBreakout::detailedCode) },
             CsvColumn("O*Net Job Codes") { prepareJobCodePart(it.rs.jobCodes, JobCodeBreakout::jobRoleCode) },
-            CsvColumn("Employers") { it.rs.employers.map { keyword -> keyword.value ?: "" }.joinToString(listDelimiter) }
+            CsvColumn("Employers") { it.rs.employers.joinToString(listDelimiter) { keyword -> keyword.value ?: "" } }
         )
         val alignmentCount = data.map { s -> s.rs.alignments.size }.maxOrNull() ?: 0
         val alignmentColumns = (0 until alignmentCount).flatMap { i ->