feat(data-platforms): adding rest resource for /dataPlatforms and mid…

…-tier support (#1817)

* feat(data-platforms): Adding rest resource for /dataPlatforms and mid-tier support

* Removed data platforms which are Linkedin internal

datahub-dao/src/main/java/com/linkedin/datahub/dao/DaoFactory.java

@@ -1,5 +1,6 @@
 package com.linkedin.datahub.dao;
 
+import com.linkedin.datahub.dao.table.DataPlatformsDao;
 import com.linkedin.datahub.dao.table.DatasetOwnerDao;
 import com.linkedin.datahub.dao.table.DatasetsDao;
 import com.linkedin.datahub.dao.table.GmsDao;
@@ -26,6 +27,7 @@ public class DaoFactory {
   private static DatasetOwnerDao datasetOwnerDao;
   private static DatasetsDao datasetsDao;
   private static LineageDao lineageDao;
+  private static DataPlatformsDao dataPlatformsDao;
 
   private DaoFactory() {
   }
@@ -101,4 +103,11 @@ public static LineageDao getLineageDao() {
     }
     return lineageDao;
   }
+
+  public static DataPlatformsDao getDataPlatformsDao() {
+    if (dataPlatformsDao == null) {
+      dataPlatformsDao = new DataPlatformsDao(getGmsDao().get_dataPlatforms());
+    }
+    return dataPlatformsDao;
+  }
 }

datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DataPlatformsDao.java

@@ -0,0 +1,25 @@
+package com.linkedin.datahub.dao.table;
+
+import com.linkedin.dataplatform.DataPlatformInfo;
+import com.linkedin.dataplatform.client.DataPlatforms;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+
+
+public class DataPlatformsDao {
+
+  private final DataPlatforms _dataPlatforms;
+
+  public DataPlatformsDao(@Nonnull DataPlatforms dataPlatforms) {
+    _dataPlatforms = dataPlatforms;
+  }
+
+  /**
+   * Get all data platforms
+   */
+  public List<Map<String, Object>> getAllPlatforms() throws Exception {
+    return _dataPlatforms.getAllPlatforms().stream().map(DataPlatformInfo::data).collect(Collectors.toList());
+  }
+}

datahub-dao/src/main/java/com/linkedin/datahub/dao/table/GmsDao.java

@@ -1,5 +1,6 @@
 package com.linkedin.datahub.dao.table;
 
+import com.linkedin.dataplatform.client.DataPlatforms;
 import com.linkedin.dataset.client.Datasets;
 import com.linkedin.dataset.client.Deprecations;
 import com.linkedin.dataset.client.InstitutionalMemory;
@@ -23,6 +24,7 @@ public class GmsDao {
   private final Deprecations _deprecations;
   private final Schemas _schemas;
   private final Lineages _lineages;
+  private final DataPlatforms _dataPlatforms;
 
   public GmsDao(@Nonnull Client restClient) {
     _corpUsers = new CorpUsers(restClient);
@@ -32,6 +34,7 @@ public GmsDao(@Nonnull Client restClient) {
     _deprecations = new Deprecations(restClient);
     _schemas = new Schemas(restClient);
     _lineages = new Lineages(restClient);
+    _dataPlatforms = new DataPlatforms(restClient);
   }
 
   public GmsDao(@Nonnull String restliHostName, @Nonnull int restliHostPort) {

datahub-frontend/app/controllers/api/v2/Dataset.java

@@ -6,6 +6,7 @@
 import com.linkedin.common.InstitutionalMemory;
 import com.linkedin.common.urn.CorpuserUrn;
 import com.linkedin.datahub.dao.DaoFactory;
+import com.linkedin.datahub.dao.table.DataPlatformsDao;
 import com.linkedin.datahub.dao.table.DatasetOwnerDao;
 import com.linkedin.datahub.dao.table.LineageDao;
 import com.linkedin.datahub.dao.view.DatasetViewDao;
@@ -30,6 +31,7 @@ public class Dataset extends Controller {
   private final OwnerViewDao _ownerViewDao;
   private final DatasetOwnerDao _datasetOwnerDao;
   private final LineageDao _lineageDao;
+  private final DataPlatformsDao _dataPlatformsDao;
 
   private static final JsonNode EMPTY_RESPONSE = Json.newObject();
 
@@ -38,6 +40,7 @@ public Dataset() {
     _ownerViewDao = DaoFactory.getOwnerViewDao();
     _datasetOwnerDao = DaoFactory.getDatasetOwnerDao();
     _lineageDao = DaoFactory.getLineageDao();
+    _dataPlatformsDao = DaoFactory.getDataPlatformsDao();
   }
 
   @Security.Authenticated(Secured.class)
@@ -264,4 +267,15 @@ public Result getDatasetDownstreams(@Nonnull String datasetUrn) {
     }
     return ok(Json.toJson(downstreams));
   }
+
+  @Security.Authenticated(Secured.class)
+  @Nonnull
+  public Result getDataPlatforms() {
+    try {
+      return ok(ControllerUtil.jsonNode("platforms", _dataPlatformsDao.getAllPlatforms()));
+    } catch (final Exception e) {
+      Logger.error("Fail to get data platforms", e);
+      return notFound(ControllerUtil.errorResponse(e));
+    }
+  }
 }

datahub-frontend/app/utils/ControllerUtil.java

@@ -1,8 +1,12 @@
 package utils;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.linkedin.data.template.RecordTemplate;
+import com.linkedin.datahub.util.RestliUtil;
 import com.linkedin.restli.client.RestLiResponseException;
+import java.io.IOException;
 import javax.annotation.Nonnull;
+import play.Logger;
 import play.libs.Json;
 
 
@@ -29,4 +33,26 @@ public static boolean checkErrorCode(Exception e, int statusCode) {
     return (e instanceof RestLiResponseException) && (((RestLiResponseException) e).getStatus() == statusCode);
   }
 
+  /**
+   * Creates a generic jsonNode out of a key and value which is a very common use case
+   * @param key
+   * @param value
+   * @return
+   */
+  @Nonnull
+  public static JsonNode jsonNode(@Nonnull final String key, @Nonnull final Object value) {
+    JsonNode node = null;
+    if (value instanceof RecordTemplate) {
+      try {
+        node = RestliUtil.toJsonNode((RecordTemplate) value);
+      } catch (final IOException e) {
+        Logger.error("Could not create a json", e);
+      }
+    } else if (value instanceof JsonNode) {
+      node = (JsonNode) value;
+    } else {
+      node = Json.toJson(value);
+    }
+    return Json.newObject().set(key, node);
+  }
 }

datahub-frontend/conf/routes

@@ -35,6 +35,7 @@ GET           /api/v2/datasets/:urn/owners/suggestion                         co
 GET           /api/v2/datasets/:urn/schema                                    controllers.api.v2.Dataset.getDatasetSchema(urn: String)
 GET           /api/v2/datasets/:urn/snapshot                                  controllers.api.v2.Dataset.getDatasetSnapshot(urn: String)
 GET           /api/v2/datasets/:urn/upstreams                                 controllers.api.v2.Dataset.getDatasetUpstreams(urn: String)
+GET           /api/v2/list/platforms                                          controllers.api.v2.Dataset.getDataPlatforms
 GET           /api/v2/search                                                  controllers.api.v2.Search.search()
 
 GET           /api/*path                                                      controllers.Application.apiNotFound(path)

gms/api/src/main/idl/com.linkedin.dataplatform.dataPlatforms.restspec.json

@@ -0,0 +1,23 @@
+{
+  "name" : "dataPlatforms",
+  "namespace" : "com.linkedin.dataplatform",
+  "path" : "/dataPlatforms",
+  "schema" : "com.linkedin.dataplatform.DataPlatformInfo",
+  "doc" : "Resource provides information about various data platforms.\n\ngenerated from: com.linkedin.metadata.resources.dataplatform.DataPlatforms",
+  "collection" : {
+    "identifier" : {
+      "name" : "platformName",
+      "type" : "string"
+    },
+    "supports" : [ "get", "get_all" ],
+    "methods" : [ {
+      "method" : "get"
+    }, {
+      "method" : "get_all",
+      "pagingSupported" : true
+    } ],
+    "entity" : {
+      "path" : "/dataPlatforms/{platformName}"
+    }
+  }
+}

gms/api/src/main/snapshot/com.linkedin.dataplatform.dataPlatforms.snapshot.json

@@ -0,0 +1,65 @@
+{
+  "models" : [ {
+    "type" : "record",
+    "name" : "DataPlatformInfo",
+    "namespace" : "com.linkedin.dataplatform",
+    "doc" : "Information about a data platform",
+    "fields" : [ {
+      "name" : "name",
+      "type" : "string",
+      "doc" : "Name of the data platform",
+      "validate" : {
+        "strlen" : {
+          "max" : 15
+        }
+      }
+    }, {
+      "name" : "type",
+      "type" : {
+        "type" : "enum",
+        "name" : "PlatformType",
+        "doc" : "Platform types available at LinkedIn",
+        "symbols" : [ "FILE_SYSTEM", "KEY_VALUE_STORE", "MESSAGE_BROKER", "OBJECT_STORE", "OLAP_DATASTORE", "OTHERS", "QUERY_ENGINE", "RELATIONAL_DB", "SEARCH_ENGINE" ],
+        "symbolDocs" : {
+          "FILE_SYSTEM" : "Value for a file system, e.g. hdfs",
+          "KEY_VALUE_STORE" : "Value for a key value store, e.g. espresso, voldemort",
+          "MESSAGE_BROKER" : "Value for a message broker, e.g. kafka",
+          "OBJECT_STORE" : "Value for an object store, e.g. ambry",
+          "OLAP_DATASTORE" : "Value for an OLAP datastore, e.g. pinot",
+          "OTHERS" : "Value for other platforms, e.g salesforce, dovetail",
+          "QUERY_ENGINE" : "Value for a query engine, e.g. presto",
+          "RELATIONAL_DB" : "Value for a relational database, e.g. oracle, mysql",
+          "SEARCH_ENGINE" : "Value for a search engine, e.g seas"
+        }
+      },
+      "doc" : "Platform type this data platform describes"
+    }, {
+      "name" : "datasetNameDelimiter",
+      "type" : "string",
+      "doc" : "The delimiter in the dataset names on the data platform, e.g. '/' for HDFS and '.' for Oracle"
+    } ]
+  }, "com.linkedin.dataplatform.PlatformType" ],
+  "schema" : {
+    "name" : "dataPlatforms",
+    "namespace" : "com.linkedin.dataplatform",
+    "path" : "/dataPlatforms",
+    "schema" : "com.linkedin.dataplatform.DataPlatformInfo",
+    "doc" : "Resource provides information about various data platforms.\n\ngenerated from: com.linkedin.metadata.resources.dataplatform.DataPlatforms",
+    "collection" : {
+      "identifier" : {
+        "name" : "platformName",
+        "type" : "string"
+      },
+      "supports" : [ "get", "get_all" ],
+      "methods" : [ {
+        "method" : "get"
+      }, {
+        "method" : "get_all",
+        "pagingSupported" : true
+      } ],
+      "entity" : {
+        "path" : "/dataPlatforms/{platformName}"
+      }
+    }
+  }
+}

gms/client/src/main/java/com/linkedin/dataplatform/client/DataPlatforms.java

@@ -0,0 +1,44 @@
+package com.linkedin.dataplatform.client;
+
+import com.linkedin.metadata.restli.BaseClient;
+import com.linkedin.dataplatform.DataPlatformInfo;
+import com.linkedin.dataplatform.DataPlatformsRequestBuilders;
+import com.linkedin.r2.RemoteInvocationException;
+import com.linkedin.restli.client.Client;
+import com.linkedin.restli.client.GetAllRequest;
+import com.linkedin.restli.client.Request;
+import java.util.List;
+import javax.annotation.Nonnull;
+
+
+public class DataPlatforms extends BaseClient {
+
+  private static final DataPlatformsRequestBuilders PLATFORMS_REQUEST_BUILDERS = new DataPlatformsRequestBuilders();
+
+  public DataPlatforms(@Nonnull Client restliClient) {
+    super(restliClient);
+  }
+
+  /**
+   * Get data platform details by name
+   * @param platformName String
+   * @return DataPlatformInfo
+   * @throws RemoteInvocationException
+   */
+  @Nonnull
+  public DataPlatformInfo getPlatformByName(@Nonnull String platformName) throws RemoteInvocationException {
+    Request<DataPlatformInfo> req = PLATFORMS_REQUEST_BUILDERS.get().id(platformName).build();
+    return _client.sendRequest(req).getResponse().getEntity();
+  }
+
+  /**
+   * Get all data platforms
+   * @return List<DataPlatformInfo>
+   * @throws RemoteInvocationException
+   */
+  @Nonnull
+  public List<DataPlatformInfo> getAllPlatforms() throws RemoteInvocationException {
+    GetAllRequest<DataPlatformInfo> req = PLATFORMS_REQUEST_BUILDERS.getAll().build();
+    return _client.sendRequest(req).getResponse().getEntity().getElements();
+  }
+}

gms/factories/src/main/java/com/linkedin/dataplatform/factory/DataPlatformLocalDaoFactory.java

@@ -0,0 +1,21 @@
+package com.linkedin.dataplatform.factory;
+
+import com.linkedin.metadata.aspect.DataPlatformAspect;
+import com.linkedin.metadata.dao.ImmutableLocalDAO;
+import com.linkedin.metadata.resources.dataplatform.utils.DataPlatformsUtil;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+
+@Configuration
+public class DataPlatformLocalDaoFactory {
+  @Autowired
+  ApplicationContext applicationContext;
+
+  @Bean(name = "dataPlatformLocalDAO")
+  protected ImmutableLocalDAO createInstance() {
+    return new ImmutableLocalDAO<>(DataPlatformAspect.class, DataPlatformsUtil.getDataPlatformInfoMap());
+  }
+}

gms/impl/src/main/java/com/linkedin/metadata/resources/dataplatform/DataPlatforms.java

@@ -0,0 +1,40 @@
+package com.linkedin.metadata.resources.dataplatform;
+
+import com.linkedin.common.urn.DataPlatformUrn;
+import com.linkedin.dataplatform.DataPlatformInfo;
+import com.linkedin.metadata.aspect.DataPlatformAspect;
+import com.linkedin.metadata.dao.ImmutableLocalDAO;
+import com.linkedin.metadata.restli.RestliUtils;
+import com.linkedin.parseq.Task;
+import com.linkedin.restli.server.PagingContext;
+import com.linkedin.restli.server.annotations.PagingContextParam;
+import com.linkedin.restli.server.annotations.RestLiCollection;
+import com.linkedin.restli.server.annotations.RestMethod;
+import com.linkedin.restli.server.resources.CollectionResourceAsyncTemplate;
+import java.util.List;
+import javax.annotation.Nonnull;
+import javax.inject.Inject;
+import javax.inject.Named;
+
+
+/**
+ * Resource provides information about various data platforms.
+ */
+@RestLiCollection(name = "dataPlatforms", namespace = "com.linkedin.dataplatform", keyName = "platformName")
+public class DataPlatforms extends CollectionResourceAsyncTemplate<String, DataPlatformInfo> {
+
+  @Inject
+  @Named("dataPlatformLocalDAO")
+  private ImmutableLocalDAO<DataPlatformAspect, DataPlatformUrn> _localDAO;
+
+  @RestMethod.Get
+  public Task<DataPlatformInfo> get(@Nonnull final String name) {
+    return RestliUtils.toTaskFromOptional(() -> _localDAO.get(DataPlatformInfo.class, new DataPlatformUrn(name)));
+  }
+
+  @RestMethod.GetAll
+  public Task<List<DataPlatformInfo>> getAll(@Nonnull @PagingContextParam(defaultCount = 100) PagingContext pagingContext) {
+    return Task.value(_localDAO.list(DataPlatformInfo.class, pagingContext.getStart(), pagingContext.getCount()).getValues());
+  }
+}
+