the dev of [FEATURE]Auto reload model when cluster rebooted/node rejoin

.whitesource

@@ -0,0 +1,22 @@
+{
+  "scanSettings": {
+    "configMode": "AUTO",
+    "configExternalURL": "",
+    "projectToken": "",
+    "baseBranches": []
+  },
+  "checkRunSettings": {
+    "vulnerableCheckRunConclusionLevel": "failure",
+    "displayMode": "diff",
+    "useMendCheckNames": true
+  },
+  "issueSettings": {
+    "minSeverityLevel": "LOW",
+    "issueType": "DEPENDENCY"
+  },
+  "remediateSettings": {
+    "workflowRules": {
+      "enabled": true
+    }
+  }
+}

client/build.gradle

@@ -15,7 +15,7 @@ plugins {
 dependencies {
     implementation project(':opensearch-ml-common')
     compileOnly group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}"
-    testImplementation group: 'junit', name: 'junit', version: '4.12'
+    testImplementation group: 'junit', name: 'junit', version: '4.13.1'
     testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.4.0'
 
 }

common/build.gradle

@@ -13,7 +13,7 @@ plugins {
 dependencies {
     compileOnly group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}"
     implementation group: 'org.reflections', name: 'reflections', version: '0.9.12'
-    testImplementation group: 'junit', name: 'junit', version: '4.12'
+    testImplementation group: 'junit', name: 'junit', version: '4.13.1'
     compileOnly "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
     compileOnly "org.opensearch:common-utils:${common_utils_version}"
     testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.4.0'

common/src/main/java/org/opensearch/ml/common/CommonValue.java

@@ -26,129 +26,134 @@ public class CommonValue {
     public static String WARM_BOX_TYPE = "warm";
 
     public static final String ML_MODEL_INDEX = ".plugins-ml-model";
+    public static final String ML_MODEL_RELOAD_INDEX = ".plugins-ml-model-reload";
+    public static final Integer ML_MODEL_RELOAD_MAX_RETRY_TIMES = 2;
     public static final String ML_TASK_INDEX = ".plugins-ml-task";
     public static final Integer ML_MODEL_INDEX_SCHEMA_VERSION = 2;
     public static final Integer ML_TASK_INDEX_SCHEMA_VERSION = 1;
     public static final String USER_FIELD_MAPPING = "      \""
-            + CommonValue.USER
-            + "\": {\n"
-            + "        \"type\": \"nested\",\n"
-            + "        \"properties\": {\n"
-            + "          \"name\": {\"type\":\"text\", \"fields\":{\"keyword\":{\"type\":\"keyword\", \"ignore_above\":256}}},\n"
-            + "          \"backend_roles\": {\"type\":\"text\", \"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\n"
-            + "          \"roles\": {\"type\":\"text\", \"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\n"
-            + "          \"custom_attribute_names\": {\"type\":\"text\", \"fields\":{\"keyword\":{\"type\":\"keyword\"}}}\n"
-            + "        }\n"
-            + "      }\n";
+        + CommonValue.USER
+        + "\": {\n"
+        + "        \"type\": \"nested\",\n"
+        + "        \"properties\": {\n"
+        + "          \"name\": {\"type\":\"text\", \"fields\":{\"keyword\":{\"type\":\"keyword\", \"ignore_above\":256}}},\n"
+        + "          \"backend_roles\": {\"type\":\"text\", \"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\n"
+        + "          \"roles\": {\"type\":\"text\", \"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\n"
+        + "          \"custom_attribute_names\": {\"type\":\"text\", \"fields\":{\"keyword\":{\"type\":\"keyword\"}}}\n"
+        + "        }\n"
+        + "      }\n";
+    public static final String NODE_ID_FIELD = "node_id";
+    public static final String MODEL_LOAD_RETRY_TIMES_FIELD = "retry_times";
+
     public static final String ML_MODEL_INDEX_MAPPING = "{\n"
-            + "    \"_meta\": {\"schema_version\": "
-            + ML_MODEL_INDEX_SCHEMA_VERSION
-            + "},\n"
-            + "    \"properties\": {\n"
-            + "      \""
-            + MLModel.ALGORITHM_FIELD
-            + "\": {\"type\": \"keyword\"},\n"
-            + "      \""
-            + MLModel.MODEL_NAME_FIELD
-            + "\" : {\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\n"
-            + "      \""
-            + MLModel.OLD_MODEL_VERSION_FIELD
-            + "\" : {\"type\": \"long\"},\n"
-            + "      \""
-            + MLModel.MODEL_VERSION_FIELD
-            + "\" : {\"type\": \"keyword\"},\n"
-            + "      \""
-            + MLModel.MODEL_CONTENT_FIELD
-            + "\" : {\"type\": \"binary\"},\n"
-            + "      \""
-            + MLModel.CHUNK_NUMBER_FIELD
-            + "\" : {\"type\": \"long\"},\n"
-            + "      \""
-            + MLModel.TOTAL_CHUNKS_FIELD
-            + "\" : {\"type\": \"long\"},\n"
-            + "      \""
-            + MLModel.MODEL_ID_FIELD
-            + "\" : {\"type\": \"keyword\"},\n"
-            + "      \""
-            + MLModel.DESCRIPTION_FIELD
-            + "\" : {\"type\": \"text\"},\n"
-            + "      \""
-            + MLModel.MODEL_FORMAT_FIELD
-            + "\" : {\"type\": \"keyword\"},\n"
-            + "      \""
-            + MLModel.MODEL_STATE_FIELD
-            + "\" : {\"type\": \"keyword\"},\n"
-            + "      \""
-            + MLModel.MODEL_CONTENT_SIZE_IN_BYTES_FIELD
-            + "\" : {\"type\": \"long\"},\n"
-            + "      \""
-            + MLModel.MODEL_CONFIG_FIELD
-            + "\" : {\"properties\":{\""
-            + MODEL_TYPE_FIELD + "\":{\"type\":\"keyword\"},\""
-            + EMBEDDING_DIMENSION_FIELD + "\":{\"type\":\"integer\"},\""
-            + FRAMEWORK_TYPE_FIELD + "\":{\"type\":\"keyword\"},\""
-            + ALL_CONFIG_FIELD + "\":{\"type\":\"text\"}}},\n"
-            + "      \""
-            + MLModel.MODEL_CONTENT_HASH_VALUE_FIELD
-            + "\" : {\"type\": \"keyword\"},\n"
-            + "      \""
-            + MLModel.CREATED_TIME_FIELD
-            + "\": {\"type\": \"date\", \"format\": \"strict_date_time||epoch_millis\"},\n"
-            + "      \""
-            + MLModel.LAST_UPLOADED_TIME_FIELD
-            + "\": {\"type\": \"date\", \"format\": \"strict_date_time||epoch_millis\"},\n"
-            + "      \""
-            + MLModel.LAST_LOADED_TIME_FIELD
-            + "\": {\"type\": \"date\", \"format\": \"strict_date_time||epoch_millis\"},\n"
-            + "      \""
-            + MLModel.LAST_UNLOADED_TIME_FIELD
-            + "\": {\"type\": \"date\", \"format\": \"strict_date_time||epoch_millis\"},\n"
-            + USER_FIELD_MAPPING
-            + "    }\n"
-            + "}";
+        + "    \"_meta\": {\"schema_version\": "
+        + ML_MODEL_INDEX_SCHEMA_VERSION
+        + "},\n"
+        + "    \"properties\": {\n"
+        + "      \""
+        + MLModel.ALGORITHM_FIELD
+        + "\": {\"type\": \"keyword\"},\n"
+        + "      \""
+        + MLModel.MODEL_NAME_FIELD
+        + "\" : {\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\n"
+        + "      \""
+        + MLModel.OLD_MODEL_VERSION_FIELD
+        + "\" : {\"type\": \"long\"},\n"
+        + "      \""
+        + MLModel.MODEL_VERSION_FIELD
+        + "\" : {\"type\": \"keyword\"},\n"
+        + "      \""
+        + MLModel.MODEL_CONTENT_FIELD
+        + "\" : {\"type\": \"binary\"},\n"
+        + "      \""
+        + MLModel.CHUNK_NUMBER_FIELD
+        + "\" : {\"type\": \"long\"},\n"
+        + "      \""
+        + MLModel.TOTAL_CHUNKS_FIELD
+        + "\" : {\"type\": \"long\"},\n"
+        + "      \""
+        + MLModel.MODEL_ID_FIELD
+        + "\" : {\"type\": \"keyword\"},\n"
+        + "      \""
+        + MLModel.DESCRIPTION_FIELD
+        + "\" : {\"type\": \"text\"},\n"
+        + "      \""
+        + MLModel.MODEL_FORMAT_FIELD
+        + "\" : {\"type\": \"keyword\"},\n"
+        + "      \""
+        + MLModel.MODEL_STATE_FIELD
+        + "\" : {\"type\": \"keyword\"},\n"
+        + "      \""
+        + MLModel.MODEL_CONTENT_SIZE_IN_BYTES_FIELD
+        + "\" : {\"type\": \"long\"},\n"
+        + "      \""
+        + MLModel.MODEL_CONFIG_FIELD
+        + "\" : {\"properties\":{\""
+        + MODEL_TYPE_FIELD + "\":{\"type\":\"keyword\"},\""
+        + EMBEDDING_DIMENSION_FIELD + "\":{\"type\":\"integer\"},\""
+        + FRAMEWORK_TYPE_FIELD + "\":{\"type\":\"keyword\"},\""
+        + ALL_CONFIG_FIELD + "\":{\"type\":\"text\"}}},\n"
+        + "      \""
+        + MLModel.MODEL_CONTENT_HASH_VALUE_FIELD
+        + "\" : {\"type\": \"keyword\"},\n"
+        + "      \""
+        + MLModel.CREATED_TIME_FIELD
+        + "\": {\"type\": \"date\", \"format\": \"strict_date_time||epoch_millis\"},\n"
+        + "      \""
+        + MLModel.LAST_UPLOADED_TIME_FIELD
+        + "\": {\"type\": \"date\", \"format\": \"strict_date_time||epoch_millis\"},\n"
+        + "      \""
+        + MLModel.LAST_LOADED_TIME_FIELD
+        + "\": {\"type\": \"date\", \"format\": \"strict_date_time||epoch_millis\"},\n"
+        + "      \""
+        + MLModel.LAST_UNLOADED_TIME_FIELD
+        + "\": {\"type\": \"date\", \"format\": \"strict_date_time||epoch_millis\"},\n"
+        + USER_FIELD_MAPPING
+        + "    }\n"
+        + "}";
 
     public static final String ML_TASK_INDEX_MAPPING = "{\n"
-            + "    \"_meta\": {\"schema_version\": "
-            + ML_TASK_INDEX_SCHEMA_VERSION
-            + "},\n"
-            + "    \"properties\": {\n"
-            + "      \""
-            + MLTask.MODEL_ID_FIELD
-            + "\": {\"type\": \"keyword\"},\n"
-            + "      \""
-            + MLTask.TASK_TYPE_FIELD
-            + "\": {\"type\": \"keyword\"},\n"
-            + "      \""
-            + MLTask.FUNCTION_NAME_FIELD
-            + "\": {\"type\": \"keyword\"},\n"
-            + "      \""
-            + MLTask.STATE_FIELD
-            + "\": {\"type\": \"keyword\"},\n"
-            + "      \""
-            + MLTask.INPUT_TYPE_FIELD
-            + "\": {\"type\": \"keyword\"},\n"
-            + "      \""
-            + MLTask.PROGRESS_FIELD
-            + "\": {\"type\": \"float\"},\n"
-            + "      \""
-            + MLTask.OUTPUT_INDEX_FIELD
-            + "\": {\"type\": \"keyword\"},\n"
-            + "      \""
-            + MLTask.WORKER_NODE_FIELD
-            + "\": {\"type\": \"keyword\"},\n"
-            + "      \""
-            + MLTask.CREATE_TIME_FIELD
-            + "\": {\"type\": \"date\", \"format\": \"strict_date_time||epoch_millis\"},\n"
-            + "      \""
-            + MLTask.LAST_UPDATE_TIME_FIELD
-            + "\": {\"type\": \"date\", \"format\": \"strict_date_time||epoch_millis\"},\n"
-            + "      \""
-            + MLTask.ERROR_FIELD
-            + "\": {\"type\": \"text\"},\n"
-            + "      \""
-            + MLTask.IS_ASYNC_TASK_FIELD
-            + "\" : {\"type\" : \"boolean\"}, \n"
-            + USER_FIELD_MAPPING
-            + "    }\n"
-            + "}";
+        + "    \"_meta\": {\"schema_version\": "
+        + ML_TASK_INDEX_SCHEMA_VERSION
+        + "},\n"
+        + "    \"properties\": {\n"
+        + "      \""
+        + MLTask.MODEL_ID_FIELD
+        + "\": {\"type\": \"keyword\"},\n"
+        + "      \""
+        + MLTask.TASK_TYPE_FIELD
+        + "\": {\"type\": \"keyword\"},\n"
+        + "      \""
+        + MLTask.FUNCTION_NAME_FIELD
+        + "\": {\"type\": \"keyword\"},\n"
+        + "      \""
+        + MLTask.STATE_FIELD
+        + "\": {\"type\": \"keyword\"},\n"
+        + "      \""
+        + MLTask.INPUT_TYPE_FIELD
+        + "\": {\"type\": \"keyword\"},\n"
+        + "      \""
+        + MLTask.PROGRESS_FIELD
+        + "\": {\"type\": \"float\"},\n"
+        + "      \""
+        + MLTask.OUTPUT_INDEX_FIELD
+        + "\": {\"type\": \"keyword\"},\n"
+        + "      \""
+        + MLTask.WORKER_NODE_FIELD
+        + "\": {\"type\": \"keyword\"},\n"
+        + "      \""
+        + MLTask.CREATE_TIME_FIELD
+        + "\": {\"type\": \"date\", \"format\": \"strict_date_time||epoch_millis\"},\n"
+        + "      \""
+        + MLTask.LAST_UPDATE_TIME_FIELD
+        + "\": {\"type\": \"date\", \"format\": \"strict_date_time||epoch_millis\"},\n"
+        + "      \""
+        + MLTask.ERROR_FIELD
+        + "\": {\"type\": \"text\"},\n"
+        + "      \""
+        + MLTask.IS_ASYNC_TASK_FIELD
+        + "\" : {\"type\" : \"boolean\"}, \n"
+        + USER_FIELD_MAPPING
+        + "    }\n"
+        + "}";
 }

ml-algorithms/build.gradle

@@ -29,7 +29,7 @@ dependencies {
     implementation group: 'io.protostuff', name: 'protostuff-runtime', version: '1.8.0'
     implementation group: 'io.protostuff', name: 'protostuff-api', version: '1.8.0'
     implementation group: 'io.protostuff', name: 'protostuff-collectionschema', version: '1.8.0'
-    testImplementation group: 'junit', name: 'junit', version: '4.12'
+    testImplementation group: 'junit', name: 'junit', version: '4.13.1'
     testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.4.0'
     testImplementation group: 'org.mockito', name: 'mockito-inline', version: '4.4.0'
     implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre'

plugin/build.gradle

@@ -3,9 +3,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import java.util.concurrent.Callable
+
 import org.opensearch.gradle.test.RestIntegTestTask
-import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 plugins {
     id 'java'