[Serve] Make Java Replica Extendable (ray-project#19463)

java/serve/src/main/java/io/ray/serve/Constants.java

@@ -6,8 +6,8 @@
 /** Ray Serve common constants. */
 public class Constants {
 
-  /** Name of backend reconfiguration method implemented by user. */
-  public static final String BACKEND_RECONFIGURE_METHOD = "reconfigure";
+  /** Name of deployment reconfiguration method implemented by user. */
+  public static final String RECONFIGURE_METHOD = "reconfigure";
 
   /** Default histogram buckets for latency tracker. */
   public static final List<Double> DEFAULT_LATENCY_BUCKET_MS =
@@ -19,7 +19,9 @@ public class Constants {
 
   public static final String SERVE_CONTROLLER_NAME = "SERVE_CONTROLLER_ACTOR";
 
-  public static final String DEFAULT_CALL_METHOD = "call";
+  public static final String CALL_METHOD = "call";
 
   public static final String UTF8 = "UTF-8";
+
+  public static final String CHECK_HEALTH_METHOD = "checkHealth";
 }

java/serve/src/main/java/io/ray/serve/DeploymentConfig.java

@@ -0,0 +1,87 @@
+package io.ray.serve;
+
+import com.google.common.base.Preconditions;
+import java.io.Serializable;
+
+public class DeploymentConfig implements Serializable {
+
+  private static final long serialVersionUID = 4037621960087621036L;
+
+  private int numReplicas = 1;
+
+  private int maxConcurrentQueries = 100;
+
+  private Object userConfig;
+
+  private double gracefulShutdownWaitLoopS = 2;
+
+  private double gracefulShutdownTimeoutS = 20;
+
+  private boolean isCrossLanguage;
+
+  private int deploymentLanguage = 1;
+
+  public int getNumReplicas() {
+    return numReplicas;
+  }
+
+  public DeploymentConfig setNumReplicas(int numReplicas) {
+    this.numReplicas = numReplicas;
+    return this;
+  }
+
+  public int getMaxConcurrentQueries() {
+    return maxConcurrentQueries;
+  }
+
+  public DeploymentConfig setMaxConcurrentQueries(int maxConcurrentQueries) {
+    Preconditions.checkArgument(maxConcurrentQueries >= 0, "max_concurrent_queries must be >= 0");
+    this.maxConcurrentQueries = maxConcurrentQueries;
+    return this;
+  }
+
+  public Object getUserConfig() {
+    return userConfig;
+  }
+
+  public DeploymentConfig setUserConfig(Object userConfig) {
+    this.userConfig = userConfig;
+    return this;
+  }
+
+  public double getGracefulShutdownWaitLoopS() {
+    return gracefulShutdownWaitLoopS;
+  }
+
+  public DeploymentConfig setGracefulShutdownWaitLoopS(double gracefulShutdownWaitLoopS) {
+    this.gracefulShutdownWaitLoopS = gracefulShutdownWaitLoopS;
+    return this;
+  }
+
+  public double getGracefulShutdownTimeoutS() {
+    return gracefulShutdownTimeoutS;
+  }
+
+  public DeploymentConfig setGracefulShutdownTimeoutS(double gracefulShutdownTimeoutS) {
+    this.gracefulShutdownTimeoutS = gracefulShutdownTimeoutS;
+    return this;
+  }
+
+  public boolean isCrossLanguage() {
+    return isCrossLanguage;
+  }
+
+  public DeploymentConfig setCrossLanguage(boolean isCrossLanguage) {
+    this.isCrossLanguage = isCrossLanguage;
+    return this;
+  }
+
+  public int getDeploymentLanguage() {
+    return deploymentLanguage;
+  }
+
+  public DeploymentConfig setDeploymentLanguage(int deploymentLanguage) {
+    this.deploymentLanguage = deploymentLanguage;
+    return this;
+  }
+}

java/serve/src/main/java/io/ray/serve/DeploymentInfo.java

@@ -1,38 +1,75 @@
 package io.ray.serve;
 
 import java.io.Serializable;
+import java.util.Map;
 
 public class DeploymentInfo implements Serializable {
 
-  private static final long serialVersionUID = -4198364411759931955L;
+  private static final long serialVersionUID = -7132135316463505391L;
 
-  private byte[] deploymentConfig;
+  private String name;
 
-  private ReplicaConfig replicaConfig;
+  private String deploymentDef;
 
-  private byte[] deploymentVersion;
+  private Object[] initArgs;
 
-  public byte[] getDeploymentConfig() {
-    return deploymentConfig;
+  private DeploymentConfig deploymentConfig;
+
+  private DeploymentVersion deploymentVersion;
+
+  private Map<String, String> config;
+
+  public String getName() {
+    return name;
   }
 
-  public void setDeploymentConfig(byte[] deploymentConfig) {
-    this.deploymentConfig = deploymentConfig;
+  public DeploymentInfo setName(String name) {
+    this.name = name;
+    return this;
   }
 
-  public ReplicaConfig getReplicaConfig() {
-    return replicaConfig;
+  public String getDeploymentDef() {
+    return deploymentDef;
   }
 
-  public void setReplicaConfig(ReplicaConfig replicaConfig) {
-    this.replicaConfig = replicaConfig;
+  public DeploymentInfo setDeploymentDef(String deploymentDef) {
+    this.deploymentDef = deploymentDef;
+    return this;
   }
 
-  public byte[] getDeploymentVersion() {
+  public Object[] getInitArgs() {
+    return initArgs;
+  }
+
+  public DeploymentInfo setInitArgs(Object[] initArgs) {
+    this.initArgs = initArgs;
+    return this;
+  }
+
+  public DeploymentConfig getDeploymentConfig() {
+    return deploymentConfig;
+  }
+
+  public DeploymentInfo setDeploymentConfig(DeploymentConfig deploymentConfig) {
+    this.deploymentConfig = deploymentConfig;
+    return this;
+  }
+
+  public DeploymentVersion getDeploymentVersion() {
     return deploymentVersion;
   }
 
-  public void setDeploymentVersion(byte[] deploymentVersion) {
+  public DeploymentInfo setDeploymentVersion(DeploymentVersion deploymentVersion) {
     this.deploymentVersion = deploymentVersion;
+    return this;
+  }
+
+  public Map<String, String> getConfig() {
+    return config;
+  }
+
+  public DeploymentInfo setConfig(Map<String, String> config) {
+    this.config = config;
+    return this;
   }
 }

java/serve/src/main/java/io/ray/serve/DeploymentVersion.java

@@ -0,0 +1,46 @@
+package io.ray.serve;
+
+import java.io.Serializable;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.StringUtils;
+
+public class DeploymentVersion implements Serializable {
+
+  private static final long serialVersionUID = 3400261981775851058L;
+
+  private String codeVersion;
+
+  private Object userConfig;
+
+  private boolean unversioned;
+
+  public DeploymentVersion() {
+    this(null, null);
+  }
+
+  public DeploymentVersion(String codeVersion) {
+    this(codeVersion, null);
+  }
+
+  public DeploymentVersion(String codeVersion, Object userConfig) {
+    if (StringUtils.isBlank(codeVersion)) {
+      this.unversioned = true;
+      this.codeVersion = RandomStringUtils.randomAlphabetic(6);
+    } else {
+      this.codeVersion = codeVersion;
+    }
+    this.userConfig = userConfig;
+  }
+
+  public String getCodeVersion() {
+    return codeVersion;
+  }
+
+  public Object getUserConfig() {
+    return userConfig;
+  }
+
+  public boolean isUnversioned() {
+    return unversioned;
+  }
+}

java/serve/src/main/java/io/ray/serve/DummyReplica.java

@@ -0,0 +1,21 @@
+package io.ray.serve;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class DummyReplica {
+
+  private AtomicInteger counter = new AtomicInteger();
+
+  public String call() {
+    return String.valueOf(counter.incrementAndGet());
+  }
+
+  public void reconfigure(Object userConfig) {
+    counter.set(0);
+  }
+
+  public void reconfigure(Map<String, String> userConfig) {
+    counter.set(Integer.valueOf(userConfig.get("value")));
+  }
+}

java/serve/src/main/java/io/ray/serve/ProxyActor.java

@@ -137,8 +137,8 @@ public void updateRoutes(Object endpoints) {
     this.proxyRouter.updateRoutes(endpointInfos);
   }
 
-  public void ready() {
-    return;
+  public boolean ready() {
+    return true;
   }
 
   public void blockUntilEndpointExists(String endpoint, double timeoutS) {

java/serve/src/main/java/io/ray/serve/RayServeConfig.java

@@ -1,6 +1,35 @@
 package io.ray.serve;
 
-public class RayServeConfig {
+import java.io.Serializable;
+import java.util.Map;
+
+public class RayServeConfig implements Serializable {
+
+  private static final long serialVersionUID = 5367425336296141588L;
 
   public static final String PROXY_CLASS = "ray.serve.proxy.class";
+
+  public static final String METRICS_ENABLED = "ray.serve.metrics.enabled";
+
+  private String name;
+
+  private Map<String, String> config;
+
+  public String getName() {
+    return name;
+  }
+
+  public RayServeConfig setName(String name) {
+    this.name = name;
+    return this;
+  }
+
+  public Map<String, String> getConfig() {
+    return config;
+  }
+
+  public RayServeConfig setConfig(Map<String, String> config) {
+    this.config = config;
+    return this;
+  }
 }

java/serve/src/main/java/io/ray/serve/RayServeHandle.java

@@ -49,7 +49,7 @@ public RayServeHandle(
    * Returns a Ray ObjectRef whose results can be waited for or retrieved using ray.wait or ray.get
    * (or ``await object_ref``), respectively.
    *
-   * @param parameters The input parameters of the specified method to invoke on the backend.
+   * @param parameters The input parameters of the specified method to be invoked in the deployment.
    * @return ray.ObjectRef
    */
   public ObjectRef<Object> remote(Object[] parameters) {
@@ -58,7 +58,7 @@ public ObjectRef<Object> remote(Object[] parameters) {
     requestMetadata.setRequestId(RandomStringUtils.randomAlphabetic(10));
     requestMetadata.setEndpoint(endpointName);
     requestMetadata.setCallMethod(
-        handleOptions != null ? handleOptions.getMethodName() : Constants.DEFAULT_CALL_METHOD);
+        handleOptions != null ? handleOptions.getMethodName() : Constants.CALL_METHOD);
     return router.assignRequest(requestMetadata.build(), parameters);
   }