Issue #489: Extend QbeastSnapshot Interface to Expose Relevant Information

core/src/main/scala/io/qbeast/core/model/QbeastSnapshot.scala

@@ -42,7 +42,7 @@ trait QbeastSnapshot {
   /**
    * Returns the total number of data files in the snapshot.
    */
-  def allFilesCount: Long
+  def numOfFiles: Long
 
   /**
    * Provides the schema of the dataset for this snapshot.
@@ -55,6 +55,12 @@ trait QbeastSnapshot {
    */
   def loadDescription: String
 
+  /**
+   * The current table configuration.
+   * @return
+   */
+  def loadConfiguration: Map[String, String]
+
   /**
    * The current table properties of the snapshot.
    * @return

core/src/main/scala/io/qbeast/spark/index/strategies/SamplingListFilesStrategy.scala

@@ -59,7 +59,7 @@ case class SamplingListFilesStrategy(snapshot: QbeastSnapshot)
     {
       val context = SparkSession.active.sparkContext
       val execId = context.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
-      val count = snapshot.allFilesCount
+      val count = snapshot.numOfFiles
       val filteredCount = count - files.length
       val filteredPercent = (filteredCount.toDouble / count) * 100.0
       val info = f"$filteredCount of $count ($filteredPercent%.2f%%)"

delta/src/main/scala/io/qbeast/spark/delta/DeltaQbeastSnapshot.scala

@@ -46,8 +46,9 @@ case class DeltaQbeastSnapshot(tableID: QTableID) extends QbeastSnapshot with De
 
   override val basePath: Path = new Path(tableID.id)
 
-  override val snapshot: Snapshot =
-    DeltaLog.forTable(SparkSession.active, tableID.id).update()
+  private val deltaLog: DeltaLog = DeltaLog.forTable(SparkSession.active, tableID.id)
+
+  override val snapshot: Snapshot = deltaLog.update()
 
   /**
    * The current state of the snapshot.
@@ -58,7 +59,7 @@ case class DeltaQbeastSnapshot(tableID: QTableID) extends QbeastSnapshot with De
 
   override lazy val schema: StructType = snapshot.metadata.schema
 
-  override lazy val allFilesCount: Long = snapshot.allFiles.count
+  override lazy val numOfFiles: Long = snapshot.numOfFiles
 
   private val metadataMap: Map[String, String] = snapshot.metadata.configuration
 
@@ -78,6 +79,12 @@ case class DeltaQbeastSnapshot(tableID: QTableID) extends QbeastSnapshot with De
    */
   override def loadDescription: String = snapshot.metadata.description
 
+  /**
+   * The current table configuration.
+   * @return
+   */
+  override def loadConfiguration: Map[String, String] = metadataMap
+
   /**
    * Constructs revision dictionary
    *
@@ -213,10 +220,10 @@ case class DeltaQbeastSnapshot(tableID: QTableID) extends QbeastSnapshot with De
 
   /**
    * Loads the dataset of qbeast blocks from index files
-   * @param indexFile
+   * @param indexFiles
    *   A dataset of index files
    * @return
-   *   the Datasetframe
+   *   the DataFrame
    */
 
   override def loadDataframeFromIndexFiles(indexFiles: Dataset[IndexFile]): DataFrame = {

src/test/scala/io/qbeast/spark/utils/ConvertToQbeastTest.scala → src/test/scala/io/qbeast/spark/delta/ConvertToQbeastDeltaTest.scala

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.qbeast.spark.utils
+package io.qbeast.spark.delta
 
 import io.qbeast.core.model.StagingUtils
 import io.qbeast.internal.commands.ConvertToQbeastCommand
@@ -25,7 +25,7 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.SparkSession
 import org.scalatest.PrivateMethodTester
 
-class ConvertToQbeastTest
+class ConvertToQbeastDeltaTest
     extends QbeastIntegrationTestSpec
     with PrivateMethodTester
     with StagingUtils {

src/test/scala/io/qbeast/spark/delta/MetadataWriterTest.scala → src/test/scala/io/qbeast/spark/delta/MetadataWriterDeltaTest.scala

@@ -26,7 +26,7 @@ import org.apache.spark.sql.delta.DeltaLog
 import org.apache.spark.sql.delta.OptimisticTransaction
 import org.apache.spark.sql.types.StructType
 
-class MetadataWriterTest(
+class MetadataWriterDeltaTest(
     tableID: QTableID,
     mode: WriteModeValue,
     deltaLog: DeltaLog,
@@ -45,15 +45,15 @@ class MetadataWriterTest(
 
 }
 
-object MetadataWriterTest {
+object MetadataWriterDeltaTest {
 
   def apply(
       tableID: QTableID,
       mode: WriteModeValue,
       deltaLog: DeltaLog,
       options: QbeastOptions,
-      schema: StructType): MetadataWriterTest = {
-    new MetadataWriterTest(tableID, mode, deltaLog, options, schema)
+      schema: StructType): MetadataWriterDeltaTest = {
+    new MetadataWriterDeltaTest(tableID, mode, deltaLog, options, schema)
   }
 
 }

src/test/scala/io/qbeast/spark/utils/PreCommitHookIntegrationTest.scala → src/test/scala/io/qbeast/spark/delta/PreCommitHookDeltaTest.scala

@@ -1,4 +1,4 @@
-package io.qbeast.spark.utils
+package io.qbeast.spark.delta
 
 import io.qbeast.context.QbeastContext
 import io.qbeast.core.model.PreCommitHook
@@ -24,7 +24,7 @@ private class SimpleHook(kv: String) extends PreCommitHook {
 
 }
 
-class PreCommitHookIntegrationTest extends QbeastIntegrationTestSpec {
+class PreCommitHookDeltaTest extends QbeastIntegrationTestSpec {
 
   "PreCommitHook" should "run simple hooks and save their outputs to CommitInfo during writes" in
     withQbeastContextSparkAndTmpDir { (spark, tmpDir) =>

src/test/scala/io/qbeast/spark/utils/QbeastSchemaTest.scala → src/test/scala/io/qbeast/spark/delta/QbeastSchemaDeltaTest.scala

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.qbeast.spark.utils
+package io.qbeast.spark.delta
 
 import io.qbeast.QbeastIntegrationTestSpec
 import org.apache.spark.sql.AnalysisException
@@ -22,7 +22,7 @@ import org.apache.spark.sql.Row
 /**
  * Test for checking the correctness of the output schemas when Appending Data through INSERT INTO
  */
-class QbeastSchemaTest extends QbeastIntegrationTestSpec {
+class QbeastSchemaDeltaTest extends QbeastIntegrationTestSpec {
 
   "Qbeast" should "detect when schemas does not match on INSERT INTO" in
     withQbeastContextSparkAndTmpWarehouse((spark, _) => {

src/test/scala/io/qbeast/spark/docs/DocumentationTests.scala

@@ -150,7 +150,7 @@ class DocumentationTests extends QbeastIntegrationTestSpec {
       val qbeast_df = spark.read.format("qbeast").load(qbeast_table_path)
 
       val qbeastSnapshot = getQbeastSnapshot(qbeast_table_path)
-      val totalNumberOfFiles = qbeastSnapshot.allFilesCount
+      val totalNumberOfFiles = qbeastSnapshot.numOfFiles
 
       totalNumberOfFiles should be > 1L withClue
         "Total number of files in pushdown notebook changes to " + totalNumberOfFiles