HC detector profile response looks unreasonable

[ylwu-amzn]

Start a HC detector (one category field) until its state becomes running, then check its profile
GET _plugins/_anomaly_detection/detectors/0kL0Bn0Bwj0eBZYxmyfp/_profile?_all

The response looks not reasonable

1. "total_entities" : 10 , but "active_entities" : 12 , why total entities less than active entities?
2. Different entities has different model counts, for example error2 has 2 models on 2 nodes, but error6 has only 1 model on 1 node.

{
  "state" : "RUNNING",
  "error" : "",
  "models" : [
    {
      "model_id" : "0kL0Bn0Bwj0eBZYxmyfp_entity_error2",
      "entity" : [
        {
          "name" : "error_type",
          "value" : "error2"
        }
      ],
      "model_size_in_bytes" : 403491,
      "node_id" : "yWljVOONS2O1dVJQkDA47w"
    },
    {
      "model_id" : "0kL0Bn0Bwj0eBZYxmyfp_entity_error6",
      "entity" : [
        {
          "name" : "error_type",
          "value" : "error6"
        }
      ],
      "model_size_in_bytes" : 403491,
      "node_id" : "yWljVOONS2O1dVJQkDA47w"
    },
    {
      "model_id" : "0kL0Bn0Bwj0eBZYxmyfp_entity_error5",
      "entity" : [
        {
          "name" : "error_type",
          "value" : "error5"
        }
      ],
      "model_size_in_bytes" : 403491,
      "node_id" : "yWljVOONS2O1dVJQkDA47w"
    },
    {
      "model_id" : "0kL0Bn0Bwj0eBZYxmyfp_entity_error4",
      "entity" : [
        {
          "name" : "error_type",
          "value" : "error4"
        }
      ],
      "model_size_in_bytes" : 403491,
      "node_id" : "kK4XztTOQSaesGpus4F1Lw"
    },
    {
      "model_id" : "0kL0Bn0Bwj0eBZYxmyfp_entity_error3",
      "entity" : [
        {
          "name" : "error_type",
          "value" : "error3"
        }
      ],
      "model_size_in_bytes" : 403491,
      "node_id" : "kK4XztTOQSaesGpus4F1Lw"
    },
    {
      "model_id" : "0kL0Bn0Bwj0eBZYxmyfp_entity_error1",
      "entity" : [
        {
          "name" : "error_type",
          "value" : "error1"
        }
      ],
      "model_size_in_bytes" : 403491,
      "node_id" : "kK4XztTOQSaesGpus4F1Lw"
    },
    {
      "model_id" : "0kL0Bn0Bwj0eBZYxmyfp_entity_error8",
      "entity" : [
        {
          "name" : "error_type",
          "value" : "error8"
        }
      ],
      "model_size_in_bytes" : 403491,
      "node_id" : "kK4XztTOQSaesGpus4F1Lw"
    },
    {
      "model_id" : "0kL0Bn0Bwj0eBZYxmyfp_entity_error7",
      "entity" : [
        {
          "name" : "error_type",
          "value" : "error7"
        }
      ],
      "model_size_in_bytes" : 403491,
      "node_id" : "kK4XztTOQSaesGpus4F1Lw"
    },
    {
      "model_id" : "0kL0Bn0Bwj0eBZYxmyfp_entity_error9",
      "entity" : [
        {
          "name" : "error_type",
          "value" : "error9"
        }
      ],
      "model_size_in_bytes" : 403491,
      "node_id" : "kK4XztTOQSaesGpus4F1Lw"
    },
    {
      "model_id" : "0kL0Bn0Bwj0eBZYxmyfp_entity_error2",
      "entity" : [
        {
          "name" : "error_type",
          "value" : "error2"
        }
      ],
      "model_size_in_bytes" : 403491,
      "node_id" : "maByFsgkR3ihJMBYtOr7eA"
    },
    {
      "model_id" : "0kL0Bn0Bwj0eBZYxmyfp_entity_error7",
      "entity" : [
        {
          "name" : "error_type",
          "value" : "error7"
        }
      ],
      "model_size_in_bytes" : 403491,
      "node_id" : "maByFsgkR3ihJMBYtOr7eA"
    },
    {
      "model_id" : "0kL0Bn0Bwj0eBZYxmyfp_entity_error9",
      "entity" : [
        {
          "name" : "error_type",
          "value" : "error9"
        }
      ],
      "model_size_in_bytes" : 403491,
      "node_id" : "maByFsgkR3ihJMBYtOr7eA"
    }
  ],
  "total_size_in_bytes" : 4841892,
  "init_progress" : {
    "percentage" : "100%"
  },
  "total_entities" : 10,
  "active_entities" : 12,
  "ad_task" : {
    "ad_task" : {
      "task_id" : "4iewCH0BWgNc0oTIC7u3",
      "last_update_time" : 1636528302785,
      "started_by" : "admin",
      "error" : "",
      "state" : "FINISHED",
      "detector_id" : "0kL0Bn0Bwj0eBZYxmyfp",
      "task_progress" : 1.0,
      "init_progress" : 0.0,
      "execution_start_time" : 1636528294839,
      "execution_end_time" : 1636528302785,
      "is_latest" : true,
      "task_type" : "HISTORICAL_HC_DETECTOR",
      "coordinating_node" : "kK4XztTOQSaesGpus4F1Lw",
      "detector" : {
        "name" : "ylwu-test-hc",
        "description" : "test",
        "time_field" : "timestamp",
        "indices" : [
          "server_log"
        ],
        "filter_query" : {
          "match_all" : {
            "boost" : 1.0
          }
        },
        "detection_interval" : {
          "period" : {
            "interval" : 1,
            "unit" : "Minutes"
          }
        },
        "window_delay" : {
          "period" : {
            "interval" : 1,
            "unit" : "Minutes"
          }
        },
        "shingle_size" : 8,
        "schema_version" : 0,
        "feature_attributes" : [
          {
            "feature_id" : "0EL0Bn0Bwj0eBZYxmyc7",
            "feature_name" : "f",
            "feature_enabled" : true,
            "aggregation_query" : {
              "f" : {
                "sum" : {
                  "field" : "value"
                }
              }
            }
          }
        ],
        "ui_metadata" : {
          "features" : {
            "f" : {
              "aggregationBy" : "sum",
              "aggregationOf" : "value",
              "featureType" : "simple_aggs"
            }
          },
          "filters" : [ ]
        },
        "last_update_time" : 1636499233768,
        "category_field" : [
          "error_type"
        ],
        "user" : {
          "name" : "admin",
          "backend_roles" : [
            "admin"
          ],
          "roles" : [
            "own_index",
            "all_access"
          ],
          "custom_attribute_names" : [ ],
          "user_requested_tenant" : "__user__"
        },
        "detector_type" : "MULTI_ENTITY",
        "result_index" : "opensearch-ad-plugin-result-ylwu-hc"
      },
      "detection_date_range" : {
        "start_time" : 1636439340000,
        "end_time" : 1636525740000
      },
      "user" : {
        "name" : "admin",
        "backend_roles" : [
          "admin"
        ],
        "roles" : [
          "own_index",
          "all_access"
        ],
        "custom_attribute_names" : [ ],
        "user_requested_tenant" : "__user__"
      }
    }
  },
  "model_count" : 12
}

[ylwu-amzn]

From @kaituo , HC detector may leave stale model on old node if hash ring changed and entity routes to new node. So it's possible one entity has multiple models on different nodes.

We should remove duplicate entity model for active entities. Current "active_entities" actually is the same with "model_count"

[kaituo]

Possible improvement:

total_entities: cardinality of categorical fields in the past x intervals.
active_entities: number of entities in the past interval using cardinality aggregation.
model_count: active models

[ylwu-amzn]

@kaituo , is this issue fixed? Can we close this one?