SPARK-3276 Changed the property name and variable name for minRememberDuration

* switched back to using spark.streaming.minRememberDuration
* renamed minRememberDurationMin to minRememberDuration

Author: emres

streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala

@@ -63,7 +63,7 @@ import org.apache.spark.util.{TimeStampedHashMap, Utils}
  *   the streaming app.
  * - If a file is to be visible in the directory listings, it must be visible within a certain
  *   duration of the mod time of the file. This duration is the "remember window", which is set to
- *   1 minute (see `FileInputDStream.minRememberDurationMin`). Otherwise, the file will never be
+ *   1 minute (see `FileInputDStream.minRememberDuration`). Otherwise, the file will never be
  *   selected as the mod time will be less than the ignore threshold when it becomes visible.
  * - Once a file is visible, the mod time cannot change. If it does due to appends, then the
  *   processing semantics are undefined.
@@ -81,13 +81,13 @@ class FileInputDStream[K, V, F <: NewInputFormat[K,V]](
   private val serializableConfOpt = conf.map(new SerializableWritable(_))
 
   /**
-   * Minimum duration of remembering the information of selected files. Defaults to 1 minute.
+   * Minimum duration of remembering the information of selected files. Defaults to 60 seconds.
    *
    * Files with mod times older than this "window" of remembering will be ignored. So if new
    * files are visible within this window, then the file will get selected in the next batch.
    */
-  private val minRememberDurationMin =
-    Minutes(ssc.conf.getLong("spark.streaming.minRememberDurationMin", 1L))
+  private val minRememberDuration =
+    Seconds(ssc.conf.getLong("spark.streaming.minRememberDuration", 60L))
 
   // This is a def so that it works during checkpoint recovery:
   private def clock = ssc.scheduler.clock
@@ -105,7 +105,7 @@ class FileInputDStream[K, V, F <: NewInputFormat[K,V]](
    * selected and processed.
    */
   private val numBatchesToRemember = FileInputDStream
-    .calculateNumBatchesToRemember(slideDuration, minRememberDurationMin)
+    .calculateNumBatchesToRemember(slideDuration, minRememberDuration)
   private val durationToRemember = slideDuration * numBatchesToRemember
   remember(durationToRemember)
 
@@ -344,10 +344,10 @@ object FileInputDStream {
 
   /**
    * Calculate the number of last batches to remember, such that all the files selected in
-   * at least last minRememberDurationMin duration can be remembered.
+   * at least last minRememberDuration duration can be remembered.
    */
   def calculateNumBatchesToRemember(batchDuration: Duration,
-                                    minRememberDurationMin: Duration): Int = {
-    math.ceil(minRememberDurationMin.milliseconds.toDouble / batchDuration.milliseconds).toInt
+                                    minRememberDuration: Duration): Int = {
+    math.ceil(minRememberDuration.milliseconds.toDouble / batchDuration.milliseconds).toInt
   }
 }