KYLIN-2180 Add project config

Signed-off-by: Yang Li <liyang@apache.org>

core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java

@@ -30,7 +30,6 @@
 import com.google.common.collect.Maps;
 
 public class KylinConfigTest extends LocalFileMetadataTestCase {
-
     @Before
     public void setUp() throws Exception {
         this.createTestMetadata();
@@ -49,33 +48,33 @@ public void testMRConfigOverride() {
         assertEquals("test1", override.get("test1"));
         assertEquals("test2", override.get("test2"));
     }
-    
+
     @Test
     public void testBackwardCompatibility() {
         KylinConfig config = KylinConfig.getInstanceFromEnv();
         final String oldk = "kylin.test.bcc.old.key";
         final String newk = "kylin.test.bcc.new.key";
-        
+
         assertNull(config.getOptional(oldk));
         assertNotNull(config.getOptional(newk));
-        
+
         Map<String, String> override = Maps.newHashMap();
         override.put(oldk, "1");
         KylinConfigExt ext = KylinConfigExt.createInstance(config, override);
         assertEquals(ext.getOptional(oldk), null);
         assertEquals(ext.getOptional(newk), "1");
         assertNotEquals(config.getOptional(newk), "1");
-        
+
         config.setProperty(oldk, "2");
         assertEquals(config.getOptional(newk), "2");
     }
-    
+
     @Test
     public void testExtShareTheBase() {
         KylinConfig config = KylinConfig.getInstanceFromEnv();
         Map<String, String> override = Maps.newHashMap();
         KylinConfig configExt = KylinConfigExt.createInstance(config, override);
-        
+
         assertTrue(config.properties == configExt.properties);
         config.setProperty("1234", "1234");
         assertEquals("1234", configExt.getOptional("1234"));

core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java

@@ -529,7 +529,16 @@ public void init(KylinConfig config) {
 
         // note CubeDesc.name == CubeInstance.name
         List<ProjectInstance> ownerPrj = ProjectManager.getInstance(config).findProjects(RealizationType.CUBE, name);
-        logger.info("CubeDesc '" + name + "' is owned by " + ownerPrj);
+
+        // cube inherit the project override props
+        if (ownerPrj.size() == 1) {
+            Map<String, String> prjOverrideProps = ownerPrj.get(0).getOverrideKylinProps();
+            for (Entry<String, String> entry : prjOverrideProps.entrySet()) {
+                if (!overrideKylinProps.containsKey(entry.getKey())) {
+                    overrideKylinProps.put(entry.getKey(), entry.getValue());
+                }
+            }
+        }
 
         this.config = KylinConfigExt.createInstance(config, overrideKylinProps);
 

core-cube/src/test/java/org/apache/kylin/cube/ProjectSpecificConfigTest.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ProjectSpecificConfigTest extends LocalFileMetadataTestCase {
+
+    @Before
+    public void setUp() throws Exception {
+        this.createTestMetadata();
+    }
+
+    @After
+    public void after() throws Exception {
+        this.cleanupTestMetadata();
+    }
+
+    @Test
+    public void testProject1() {
+        KylinConfig baseConfig = KylinConfig.getInstanceFromEnv();
+        CubeDesc cubeDesc = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb");
+        verifyProjectOverride(baseConfig, cubeDesc.getConfig());
+    }
+
+    @Test
+    public void testProject2() {
+        KylinConfig baseConfig = KylinConfig.getInstanceFromEnv();
+        CubeInstance cube = CubeManager.getInstance(baseConfig).getCube("ssb");
+        verifyProjectOverride(baseConfig, cube.getConfig());
+    }
+
+    private void verifyProjectOverride(KylinConfig base, KylinConfig override) {
+        assertEquals("whoami@kylin.apache.org", base.getKylinOwner());
+        assertEquals("kylin@kylin.apache.org", override.getKylinOwner());
+    }
+}

core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java

@@ -19,12 +19,14 @@
 package org.apache.kylin.metadata.project;
 
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
 
 import javax.annotation.Nullable;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
@@ -78,6 +80,10 @@ public class ProjectInstance extends RootPersistentEntity {
     @JsonProperty("ext_filters")
     private Set<String> extFilters = new TreeSet<String>();
 
+    @JsonProperty("overrideKylinProps")
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    private LinkedHashMap<String, String> overrideKylinProps;
+
     public String getResourcePath() {
         return concatResourcePath(name);
     }
@@ -93,7 +99,7 @@ public static String getNormalizedProjectName(String project) {
         return project.toUpperCase();
     }
 
-    public static ProjectInstance create(String name, String owner, String description, List<RealizationEntry> realizationEntries, List<String> models) {
+    public static ProjectInstance create(String name, String owner, String description, LinkedHashMap<String, String> overrideProps, List<RealizationEntry> realizationEntries, List<String> models) {
         ProjectInstance projectInstance = new ProjectInstance();
 
         projectInstance.updateRandomUuid();
@@ -102,6 +108,11 @@ public static ProjectInstance create(String name, String owner, String descripti
         projectInstance.setDescription(description);
         projectInstance.setStatus(ProjectStatusEnum.ENABLED);
         projectInstance.setCreateTimeUTC(System.currentTimeMillis());
+        if (overrideProps != null) {
+            projectInstance.setOverrideKylinProps(overrideProps);
+        } else {
+            projectInstance.setOverrideKylinProps(new LinkedHashMap<String, String>());
+        }
         if (realizationEntries != null)
             projectInstance.setRealizationEntries(realizationEntries);
         else
@@ -285,6 +296,14 @@ public void removeModel(String modelName) {
         }
     }
 
+    public LinkedHashMap<String, String> getOverrideKylinProps() {
+        return overrideKylinProps;
+    }
+
+    public void setOverrideKylinProps(LinkedHashMap<String, String> overrideKylinProps) {
+        this.overrideKylinProps = overrideKylinProps;
+    }
+
     public void init() {
         if (name == null)
             name = ProjectInstance.DEFAULT_PROJECT_NAME;
@@ -296,6 +315,10 @@ public void init() {
         if (tables == null)
             tables = new TreeSet<String>();
 
+        if (overrideKylinProps == null) {
+            overrideKylinProps = new LinkedHashMap<>();
+        }
+
         if (StringUtils.isBlank(this.name))
             throw new IllegalStateException("Project name must not be blank");
     }

core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java

@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -138,7 +139,6 @@ public ProjectInstance reloadProjectLocal(String project) throws IOException {
     }
 
     private ProjectInstance reloadProjectLocalAt(String path) throws IOException {
-
         ProjectInstance projectInstance = getStore().getResource(path, ProjectInstance.class, PROJECT_SERIALIZER);
         if (projectInstance == null) {
             logger.warn("reload project at path:" + path + " not found, this:" + this.toString());
@@ -162,12 +162,12 @@ public ProjectInstance getProject(String projectName) {
         return projectMap.get(projectName);
     }
 
-    public ProjectInstance createProject(String projectName, String owner, String description) throws IOException {
+    public ProjectInstance createProject(String projectName, String owner, String description, LinkedHashMap<String, String> overrideProps) throws IOException {
         logger.info("Creating project " + projectName);
 
         ProjectInstance currentProject = getProject(projectName);
         if (currentProject == null) {
-            currentProject = ProjectInstance.create(projectName, owner, description, null, null);
+            currentProject = ProjectInstance.create(projectName, owner, description, overrideProps, null, null);
         } else {
             throw new IllegalStateException("The project named " + projectName + "already exists");
         }
@@ -207,9 +207,9 @@ public void updateProject(RealizationType type, String realizationName) throws I
     }
 
     //update project itself
-    public ProjectInstance updateProject(ProjectInstance project, String newName, String newDesc) throws IOException {
+    public ProjectInstance updateProject(ProjectInstance project, String newName, String newDesc, LinkedHashMap<String, String> overrideProps) throws IOException {
         if (!project.getName().equals(newName)) {
-            ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc);
+            ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc, overrideProps);
 
             newProject.setCreateTimeUTC(project.getCreateTimeUTC());
             newProject.recordUpdateTime(System.currentTimeMillis());
@@ -225,6 +225,7 @@ public ProjectInstance updateProject(ProjectInstance project, String newName, St
         } else {
             project.setName(newName);
             project.setDescription(newDesc);
+            project.setOverrideKylinProps(overrideProps);
 
             if (project.getUuid() == null)
                 project.updateRandomUuid();
@@ -291,7 +292,7 @@ private ProjectInstance addRealizationToProject(RealizationType type, String rea
         String newProjectName = norm(project);
         ProjectInstance newProject = getProject(newProjectName);
         if (newProject == null) {
-            newProject = this.createProject(newProjectName, user, "This is a project automatically added when adding realization " + realizationName + "(" + type + ")");
+            newProject = this.createProject(newProjectName, user, "This is a project automatically added when adding realization " + realizationName + "(" + type + ")", null);
         }
         newProject.addRealizationEntry(type, realizationName);
         updateProject(newProject);

examples/test_case_data/localmeta/project/default.json

@@ -41,6 +41,11 @@
       "name": "test_kylin_cube_with_view_inner_join_empty",
       "type": "CUBE",
       "realization": "test_kylin_cube_with_view_inner_join_empty"
+    },
+    {
+      "name": "ssb",
+      "type": "CUBE",
+      "realization": "ssb"
     }
   ],
   "tables": [
@@ -59,5 +64,8 @@
     "test_kylin_left_join_model_desc",
     "test_kylin_left_join_view_model_desc",
     "test_streaming_table_model_desc"
-  ]
+  ],
+  "overrideKylinProps" :{
+    "kylin.storage.hbase.owner-tag": "kylin@kylin.apache.org"
+  }
 }

server-base/src/main/java/org/apache/kylin/rest/request/CreateProjectRequest.java

@@ -18,11 +18,14 @@
 
 package org.apache.kylin.rest.request;
 
+import java.util.LinkedHashMap;
+
 /**
  */
 public class CreateProjectRequest {
     private String name;
     private String description;
+    private LinkedHashMap<String, String> overrideKylinProps;
 
     public CreateProjectRequest() {
     }
@@ -43,4 +46,12 @@ public void setDescription(String description) {
         this.description = description;
     }
 
+    public LinkedHashMap<String, String> getOverrideKylinProps() {
+        return overrideKylinProps;
+    }
+
+    public void setOverrideKylinProps(LinkedHashMap<String, String> overrideKylinProps) {
+        this.overrideKylinProps = overrideKylinProps;
+    }
+
 }

server-base/src/main/java/org/apache/kylin/rest/request/UpdateProjectRequest.java

@@ -18,12 +18,15 @@
 
 package org.apache.kylin.rest.request;
 
+import java.util.LinkedHashMap;
+
 /**
  */
 public class UpdateProjectRequest {
     private String formerProjectName;
     private String newProjectName;
     private String newDescription;
+    private LinkedHashMap<String, String> overrideKylinProps;
 
     public UpdateProjectRequest() {
     }
@@ -52,4 +55,12 @@ public String getNewProjectName() {
     public void setNewProjectName(String newProjectName) {
         this.newProjectName = newProjectName;
     }
+
+    public LinkedHashMap<String, String> getOverrideKylinProps() {
+        return overrideKylinProps;
+    }
+
+    public void setOverrideKylinProps(LinkedHashMap<String, String> overrideKylinProps) {
+        this.overrideKylinProps = overrideKylinProps;
+    }
 }

server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java

@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.List;
 
 import org.apache.kylin.metadata.project.ProjectInstance;
@@ -53,13 +54,15 @@ public class ProjectService extends BasicService {
     public ProjectInstance createProject(CreateProjectRequest projectRequest) throws IOException {
         String projectName = projectRequest.getName();
         String description = projectRequest.getDescription();
+        LinkedHashMap<String, String> overrideProps = projectRequest.getOverrideKylinProps();
+
         ProjectInstance currentProject = getProjectManager().getProject(projectName);
 
         if (currentProject != null) {
             throw new InternalErrorException("The project named " + projectName + " already exists");
         }
         String owner = SecurityContextHolder.getContext().getAuthentication().getName();
-        ProjectInstance createdProject = getProjectManager().createProject(projectName, owner, description);
+        ProjectInstance createdProject = getProjectManager().createProject(projectName, owner, description, overrideProps);
         accessService.init(createdProject, AclPermission.ADMINISTRATION);
         logger.debug("New project created.");
 
@@ -71,19 +74,19 @@ public ProjectInstance updateProject(UpdateProjectRequest projectRequest, Projec
         String formerProjectName = projectRequest.getFormerProjectName();
         String newProjectName = projectRequest.getNewProjectName();
         String newDescription = projectRequest.getNewDescription();
+        LinkedHashMap<String, String> overrideProps = projectRequest.getOverrideKylinProps();
 
         if (currentProject == null) {
             throw new InternalErrorException("The project named " + formerProjectName + " does not exists");
         }
 
-        ProjectInstance updatedProject = getProjectManager().updateProject(currentProject, newProjectName, newDescription);
+        ProjectInstance updatedProject = getProjectManager().updateProject(currentProject, newProjectName, newDescription, overrideProps);
 
         logger.debug("Project updated.");
 
         return updatedProject;
     }
 
-
     @PostFilter(Constant.ACCESS_POST_FILTER_READ)
     public List<ProjectInstance> listProjects(final Integer limit, final Integer offset) {
         List<ProjectInstance> projects = listAllProjects(limit, offset);