add max_lines_per_page setting to ElasticsearchTaskHandler

providers/elasticsearch/provider.yaml

@@ -206,6 +206,15 @@ config:
         type: string
         example: module.callable
         default: ""
+      max_lines_per_page:
+        description: |
+          The Page size of logs to retrieve from elasticsearch using the ElasticsearchTaskHandler.
+          If this is set too low, this can cause log read queries to become very slow.
+          This is limited on the server side (default limit of 10,000).
+        version_added: 6.4.4
+        type: string
+        example: ~
+        default: "1000"
   elasticsearch_configs:
     description: ~
     options:

providers/elasticsearch/src/airflow/providers/elasticsearch/get_provider_info.py

@@ -143,6 +143,13 @@ def get_provider_info():
                         "example": "module.callable",
                         "default": "",
                     },
+                    "max_lines_per_page": {
+                        "description": "The Page size of logs to retrieve from elasticsearch using the ElasticsearchTaskHandler.\nIf this is set too low, this can cause log read queries to become very slow.\nThis is limited on the server side (default limit of 10,000).\n",
+                        "version_added": "6.4.4",
+                        "type": "string",
+                        "example": None,
+                        "default": "1000",
+                    },
                 },
             },
             "elasticsearch_configs": {

providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py

@@ -138,7 +138,7 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
     """
 
     PAGE = 0
-    MAX_LINE_PER_PAGE = 1000
+    MAX_LINE_PER_PAGE = conf.getint("elasticsearch", "max_lines_per_page", fallback=1000)
     LOG_NAME = "Elasticsearch"
 
     trigger_should_wrap = True