Add Jinja filters for creating dynamic date ranges based on now() (#1884

)

* Add start and end date functions to serve as Jinja filters for creating custom date ranges
* Add new date range functions as Jinja filters to the track loader
* Add tests for the two Jinja date filters
* Small corrections to now and days_ago() documentation
* Add documentation section for dynamic date ranges using get_start_date() and get_end_date()

docs/advanced.rst

@@ -6,7 +6,7 @@ Advanced Topics
 Templating Track Files with Jinja2
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Rally uses `Jinja2 <http://jinja.pocoo.org/docs/dev/>`_ as a template language so you can use Jinja2 expressions in track files.
+Rally uses `Jinja2 <https://jinja.palletsprojects.com/en/latest/>`_ as a template language so you can use Jinja2 expressions in track files.
 
 Elasticsearch utilizes Mustache formatting in a few places, notably in `search templates <https://www.elastic.co/guide/en/elasticsearch/reference/7.4/search-template.html>`_ and `Watcher templates <https://www.elastic.co/guide/en/elasticsearch/reference/7.4/actions-email.html>`_. If you are using Mustache in your Rally tracks you must `escape them properly <https://jinja.palletsprojects.com/en/2.10.x/templates/#escaping>`_. See :ref:`put_pipeline` for an example.
 
@@ -17,8 +17,8 @@ Extensions
 
 Rally also provides a few extensions to Jinja2:
 
-* ``now``: a global variable that represents the current date and time when the template is evaluated by Rally.
-* ``days_ago()``: a `filter <http://jinja.pocoo.org/docs/dev/templates/#filters>`_ that you can use for date calculations.
+* ``now``: a global variable that represents the current date and time in seconds since the epoch (Jan 1, 1970 00:00:00 UTC) when the template is evaluated by Rally.
+* ``days_ago()``: a `filter <https://jinja.palletsprojects.com/en/latest/templates/#filters>`_ that you can use for date calculations.
 
 You can find an example in the ``http_logs`` track::
 
@@ -90,6 +90,45 @@ The parameter ``default_value`` controls the value to use for the setting if it
 * ``build_flavor``: a global variable that holds build flavor reported by Elasticsearch cluster targetted by Rally.
 * ``serverless_operator`` (reserved for Elastic internal use): a global variable that holds ``true`` when Rally targets serverless cluster with operator privileges, and ``false`` otherwise.
 
+Dynamic Date Ranges
+~~~~~~~~~~~~~~~~~~~
+
+* ``get_start_date('%Y-%m-%d', 'timezone.utc')``: pair with the ``now`` global variable to generate a start date in the specified date format and time zone. The default format is ``%Y-%m-%d`` and time zone is ``timezone.utc``.  The filter will use the default values when no parameters are specified.
+* ``get_end_date(1, '%Y-%m-%d', 'timezone.utc')``: pair with the ``now`` global variable to generate an end date with the specified number of days, date format, and time zone. The default number of days is 1, format is ``%Y-%m-%d``, and time zone is ``timezone.utc``. The filter will use the default values when no parameters are specified.
+
+Example 1::
+
+    {
+      ...,
+      "start-date": {{ now | get_start_date | tojson }},
+      "end-date": {{ now | get_end_date | tojson }}
+    }
+
+Results in::
+
+    {
+      ...,
+      "start-date": "2021-01-01",
+      "end-date": "2021-01-02"
+    }
+
+Example 2::
+
+    {
+      ...,
+      "start-date": {{ now | get_start_date('%Y-%m-%d') | tojson }},
+      "end-date": {{ now | get_end_date(3, '%Y-%m-%d') | tojson }}
+    }
+
+Results in::
+
+    {
+      ...,
+      "start-date": "2021-01-01",
+      "end-date": "2021-01-04"
+    }
+
+
 .. _adding_tracks_custom_param_sources:
 
 Controlling Operation Parameters Using Custom Parameter Sources

esrally/time.py

@@ -16,7 +16,7 @@
 # under the License.
 
 import time
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 
 
 def to_epoch_millis(t):
@@ -84,6 +84,33 @@ def days_ago(start_date, end_date, date_format="%d-%m-%Y", default_tz=timezone.u
     return (end - start).days
 
 
+def get_start_date(the_date, date_format="%Y-%m-%d", default_tz=timezone.utc):
+    """
+    Return a formatted date string for a given date.
+
+    :param the_date: The date to format. May be a datetime instance, a unix timestamp or a string representation of a date.
+    :param date_format: The format to use for the date. Default: "%Y-%m-%d"
+    :param default_tz: Timezone to use for dates using unix timestamp. Default: timezone.UTC
+    :return: Returns a formatted date string.
+    """
+    start_date = _to_datetime(the_date, default_tz)
+    return start_date.strftime(date_format)
+
+
+def get_end_date(start_date, duration_days=1, date_format="%Y-%m-%d", default_tz=timezone.utc):
+    """
+    Return a formatted date string for a given date plus a duration.
+
+    :param start_date: The date to start from. May be a datetime instance, a unix timestamp or a string representation of a date.
+    :param duration_days: The duration in days to add to the start date. Default: 1
+    :param date_format: The format to use for the date. Default: "%Y-%m-%d"
+    :param default_tz: Timezone to use for dates using unix timestamp. Default: timezone.UTC
+    :return: Returns a formatted date string.
+    """
+    end_date = _to_datetime(start_date, default_tz) + timedelta(days=duration_days)
+    return end_date.strftime(date_format)
+
+
 class Clock:
     """
     A clock abstracts time measurements. Its main purpose is to ease testing

esrally/track/loader.py

@@ -757,6 +757,8 @@ def default_internal_template_vars(glob_helper=lambda f: [], clock=time.Clock, b
         },
         "filters": {
             "days_ago": time.days_ago,
+            "get_start_date": time.get_start_date,
+            "get_end_date": time.get_end_date,
         },
     }
 

tests/track/loader_test.py

@@ -974,7 +974,9 @@ def test_render_simple_template(self):
             "key": {{'01-01-2000' | days_ago(now)}},
             "key2": "static value",
             "key3": "{{build_flavor}}",
-            "key4": {{serverless_operator}}
+            "key4": {{serverless_operator}},
+            "key5": {{now | get_start_date('%Y-%m-%d') | tojson}},
+            "key6": {{now | get_end_date(1, '%Y-%m-%d') | tojson}}
         }
         """
 
@@ -985,7 +987,9 @@ def test_render_simple_template(self):
             "key": 5864,
             "key2": "static value",
             "key3": "test",
-            "key4": False
+            "key4": False,
+            "key5": "2016-01-21",
+            "key6": "2016-01-22"
         }
         """
         assert rendered == expected