diff --git a/build.gradle b/build.gradle index 664bed346..378722207 100644 --- a/build.gradle +++ b/build.gradle @@ -74,7 +74,7 @@ buildscript { ext { // as we don't have 3.0.0, 2.4.0 version for K-NN on darwin we need to keep OpenSearch version as 2.3 for now. // Github issue: https://github.com/opensearch-project/opensearch-build/issues/2662 - opensearch_version = System.getProperty("opensearch.version", "2.3.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT") buildVersionQualifier = System.getProperty("build.version_qualifier", "") isSnapshot = "true" == System.getProperty("build.snapshot", "true") version_tokens = opensearch_version.tokenize('-') @@ -89,12 +89,6 @@ buildscript { } opensearch_group = "org.opensearch" opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","") - k_NN_resource_folder = "build/resources/k-NN" - ml_common_resource_folder = "build/resources/ml-commons" - //TODO: we need a better way to construct this URL as, this URL is valid for released version of K-NN, ML-Plugin. - // Github issue: https://github.com/opensearch-project/opensearch-build/issues/2662 - k_NN_build_download_url = "https://aws.oss.sonatype.org/content/repositories/releases/org/opensearch/plugin/opensearch-knn/" + opensearch_no_snapshot + "/opensearch-knn-" + opensearch_no_snapshot +".zip" - ml_common_build_download_url = "https://aws.oss.sonatype.org/content/repositories/releases/org/opensearch/plugin/opensearch-ml-plugin/" + opensearch_no_snapshot + "/opensearch-ml-plugin-" + opensearch_no_snapshot +".zip" } repositories { @@ -137,22 +131,21 @@ def knnJarDirectory = "$buildDir/dependencies/opensearch-knn" dependencies { api "org.opensearch:opensearch:${opensearch_version}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}" + zipArchive group: 'org.opensearch', name:'opensearch-ml-plugin', version: "${opensearch_build}" compileOnly fileTree(dir: knnJarDirectory, include: '*.jar') api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}" implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' } -// From maven, we can get the k-NN plugin as a zip. In order to add the jar to the classpath, we need to unzip the +// In order to add the jar to the classpath, we need to unzip the // k-NN zip and then copy it into a directory that we specify as a dependency. -task unzip(type: Copy) { - configurations.zipArchive.asFileTree.each { - from(zipTree(it)) - } +task extractKnnJar(type: Copy) { + from(zipTree(configurations.zipArchive.find { it.name.startsWith("opensearch-knn")})) into knnJarDirectory } compileJava { - dependsOn unzip + dependsOn extractKnnJar options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } compileTestJava { @@ -208,45 +201,20 @@ integTest { testClusters.integTest { testDistribution = "ARCHIVE" - // Install ML-Plugin on the integTest cluster nodes - plugin(provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - if (new File("$project.rootDir/$ml_common_resource_folder").exists()) { - project.delete(files("$project.rootDir/$ml_common_resource_folder")) - } - project.mkdir ml_common_resource_folder - ant.get(src: ml_common_build_download_url, - dest: ml_common_resource_folder, - httpusecaches: false) - return fileTree(ml_common_resource_folder).getSingleFile() - } - } - } - })) - - // Install K-NN plugin on the integTest cluster nodes - plugin(provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - if (new File("$project.rootDir/$k_NN_resource_folder").exists()) { - project.delete(files("$project.rootDir/$k_NN_resource_folder")) + // Install K-NN/ml-commons plugins on the integTest cluster nodes + configurations.zipArchive.asFileTree.each { + plugin(provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return it } - project.mkdir k_NN_resource_folder - ant.get(src: k_NN_build_download_url, - dest: k_NN_resource_folder, - httpusecaches: false) - return fileTree(k_NN_resource_folder).getSingleFile() } } - } - })) + })) + } // This installs our neural-search plugin into the testClusters plugin(project.tasks.bundlePlugin.archiveFile) diff --git a/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java b/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java index fb718e9be..68aacd2cf 100644 --- a/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java +++ b/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java @@ -22,7 +22,6 @@ import org.opensearch.ml.common.dataset.MLInputDataset; import org.opensearch.ml.common.dataset.TextDocsInputDataSet; import org.opensearch.ml.common.input.MLInput; -import org.opensearch.ml.common.model.MLModelTaskType; import org.opensearch.ml.common.output.MLOutput; import org.opensearch.ml.common.output.model.ModelResultFilter; import org.opensearch.ml.common.output.model.ModelTensor; @@ -67,7 +66,7 @@ public void inferenceSentence( /** * Abstraction to call predict function of api of MLClient with default targetResponse filters. It uses the - * custom model provided as modelId and run the {@link MLModelTaskType#TEXT_EMBEDDING}. The return will be sent + * custom model provided as modelId and run the {@link FunctionName#TEXT_EMBEDDING}. The return will be sent * using the actionListener which will have a {@link List} of {@link List} of {@link Float} in the order of * inputText. We are not making this function generic enough to take any function or TaskType as currently we * need to run only TextEmbedding tasks only. @@ -86,7 +85,7 @@ public void inferenceSentences( /** * Abstraction to call predict function of api of MLClient with provided targetResponse filters. It uses the - * custom model provided as modelId and run the {@link MLModelTaskType#TEXT_EMBEDDING}. The return will be sent + * custom model provided as modelId and run the {@link FunctionName#TEXT_EMBEDDING}. The return will be sent * using the actionListener which will have a {@link List} of {@link List} of {@link Float} in the order of * inputText. We are not making this function generic enough to take any function or TaskType as currently we * need to run only TextEmbedding tasks only. @@ -112,7 +111,7 @@ public void inferenceSentences( /** * Abstraction to call predict function of api of MLClient with provided targetResponseFilters. It uses the - * custom model provided as modelId and run the {@link MLModelTaskType#TEXT_EMBEDDING}. The return will be sent + * custom model provided as modelId and run the {@link FunctionName#TEXT_EMBEDDING}. The return will be sent * using the actionListener which will have a {@link List} of {@link List} of {@link Float} in the order of * inputText. We are not making this function generic enough to take any function or TaskType as currently we need * to run only TextEmbedding tasks only. Please note this method is a blocking method, use this only when the processing @@ -136,7 +135,7 @@ public List> inferenceSentences(@NonNull final String modelId, @NonN private MLInput createMLInput(final List targetResponseFilters, List inputText) { final ModelResultFilter modelResultFilter = new ModelResultFilter(false, true, targetResponseFilters, null); final MLInputDataset inputDataset = new TextDocsInputDataSet(inputText, modelResultFilter); - return new MLInput(FunctionName.TEXT_EMBEDDING, null, inputDataset, MLModelTaskType.TEXT_EMBEDDING); + return new MLInput(FunctionName.TEXT_EMBEDDING, null, inputDataset); } private List> buildVectorFromResponse(MLOutput mlOutput) { diff --git a/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java b/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java index b4a4f21a0..91335d970 100644 --- a/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java @@ -27,8 +27,8 @@ import org.opensearch.action.ActionListener; import org.opensearch.ml.client.MachineLearningNodeClient; import org.opensearch.ml.common.input.MLInput; -import org.opensearch.ml.common.model.MLResultDataType; import org.opensearch.ml.common.output.MLOutput; +import org.opensearch.ml.common.output.model.MLResultDataType; import org.opensearch.ml.common.output.model.ModelTensor; import org.opensearch.ml.common.output.model.ModelTensorOutput; import org.opensearch.ml.common.output.model.ModelTensors; diff --git a/src/test/resources/model/all-MiniLM-L6-v2.zip b/src/test/resources/model/all-MiniLM-L6-v2.zip index 90d46c2c1..4afc4ceda 100644 Binary files a/src/test/resources/model/all-MiniLM-L6-v2.zip and b/src/test/resources/model/all-MiniLM-L6-v2.zip differ