Skip to content

[FEATURE] Remove unnecessary filter for DataHistogram aggregation #4873

@LantaoJin

Description

@LantaoJin

Is your feature request related to a problem?
big5 query

source = big5
| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00'
| stats count() by span(`@timestamp`, 1m)

The current plan is

{
  "from": 0,
  "size": 0,
  "timeout": "1m",
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "from": "2023-01-01T00:00:00.000Z",
              "to": "2023-01-03T00:00:00.000Z",
              "include_lower": true,
              "include_upper": false,
              "format": "date_time",
              "boost": 1
            }
          }
        },
        {
          "exists": {
            "field": "@timestamp",
            "boost": 1
          }
        }
      ],
      "adjust_pure_negative": true,
      "boost": 1
    }
  },
  "_source": {
    "includes": [
      "@timestamp"
    ],
    "excludes": []
  },
  "aggregations": {
    "composite_buckets": {
      "composite": {
        "size": 10000,
        "sources": [
          {
            "span(`@timestamp`,1m)": {
              "date_histogram": {
                "field": "@timestamp",
                "missing_bucket": false,
                "order": "asc",
                "fixed_interval": "1m"
              }
            }
          }
        ]
      }
    }
  }
}

The expected plan is

{
  "from": 0,
  "size": 0,
  "timeout": "1m",
  "query": {
    "bool": {
      "range": {
      "@timestamp": {
        "from": "2023-01-01T00:00:00.000Z",
        "to": "2023-01-03T00:00:00.000Z",
        "include_lower": true,
        "include_upper": false,
        "format": "date_time",
        "boost": 1
      }
    }
  },
  "_source": {
    "includes": [
      "@timestamp"
    ],
    "excludes": []
  },
  "aggregations": {
    "composite_buckets": {
      "composite": {
        "size": 10000,
        "sources": [
          {
            "span(`@timestamp`,1m)": {
              "date_histogram": {
                "field": "@timestamp",
                "missing_bucket": false,
                "order": "asc",
                "fixed_interval": "1m"
              }
            }
          }
        ]
      }
    }
  }
}

The execution time reduced from 500ms to 20ms, which has 25x time improved.

Metadata

Metadata

Assignees

Labels

Type

No type

Projects

Status

Done

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions