Skip to content

Commit

Permalink
minor, add project to cube list api
Browse files Browse the repository at this point in the history
  • Loading branch information
@lidongsjtu @liyang-kylin
lidongsjtu authored and liyang-kylin committed Oct 17, 2017
1 parent 4a565b5 commit dcd36fa
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 26 deletions.
28 changes: 18 additions & 10 deletions server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.request.JobBuildRequest;
import org.apache.kylin.rest.request.JobBuildRequest2;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.GeneralResponse;
import org.apache.kylin.rest.response.HBaseResponse;
Expand Down Expand Up @@ -109,26 +110,34 @@ public EnvelopeResponse<Boolean> validateModelName(@PathVariable String cubeName

@RequestMapping(value = "", method = { RequestMethod.GET }, produces = { "application/json" })
@ResponseBody
public List<CubeInstance> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName,
public List<CubeInstanceResponse> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName,
@RequestParam(value = "modelName", required = false) String modelName,
@RequestParam(value = "projectName", required = false) String projectName,
@RequestParam(value = "limit", required = false) Integer limit,
@RequestParam(value = "offset", required = false) Integer offset) {
List<CubeInstance> cubes;
cubes = cubeService.listAllCubes(cubeName, projectName, modelName, true);
List<CubeInstance> cubes = cubeService.listAllCubes(cubeName, projectName, modelName, true);

int climit = (null == limit) ? cubes.size() : limit;
List<CubeInstanceResponse> response = Lists.newArrayListWithExpectedSize(cubes.size());
for (CubeInstance cube : cubes) {
try {
response.add(cubeService.createCubeInstanceResponse(cube));
} catch (Exception e) {
logger.error("Error creating cube instance response, skipping.", e);
}
}

int climit = (null == limit) ? response.size() : limit;
int coffset = (null == offset) ? 0 : offset;

if (cubes.size() <= coffset) {
if (response.size() <= coffset) {
return Collections.emptyList();
}

if ((cubes.size() - coffset) < climit) {
return cubes.subList(coffset, cubes.size());
if ((response.size() - coffset) < climit) {
return response.subList(coffset, response.size());
}

return cubes.subList(coffset, coffset + climit);
return response.subList(coffset, coffset + climit);
}

@RequestMapping(value = "validEncodings", method = { RequestMethod.GET }, produces = { "application/json" })
Expand Down Expand Up @@ -280,7 +289,7 @@ public JobInstance build(@PathVariable String cubeName, @RequestBody JobBuildReq
@RequestMapping(value = "/{cubeName}/rebuild", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public JobInstance rebuild(@PathVariable String cubeName, @RequestBody JobBuildRequest req) {
return buildInternal(cubeName, new TSRange(req.getStartTime(), req.getEndTime()), null, null, null,
return buildInternal(cubeName, new TSRange(req.getStartTime(), req.getEndTime()), null, null, null,
req.getBuildType(), req.isForce() || req.isForceMergeEmptySegment());
}

Expand Down Expand Up @@ -746,5 +755,4 @@ public void setCubeService(CubeService cubeService) {
public void setJobService(JobService jobService) {
this.jobService = jobService;
}

}
27 changes: 26 additions & 1 deletion server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.draft.Draft;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.project.ProjectInstance;
Expand All @@ -55,6 +56,7 @@
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
import org.apache.kylin.rest.request.MetricsRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.security.AclPermission;
Expand All @@ -69,6 +71,7 @@
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;

import com.google.common.base.Preconditions;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
Expand Down Expand Up @@ -446,7 +449,7 @@ public HBaseResponse getHTableInfo(String cubeName, String tableName) throws IOE
if ("hbase".equals(getConfig().getMetadataUrl().getScheme())) {
try {
logger.debug("Loading HTable info " + cubeName + ", " + tableName);

// use reflection to isolate NoClassDef errors when HBase is not available
hr = (HBaseResponse) Class.forName("org.apache.kylin.rest.service.HBaseInfoUtil")//
.getMethod("getHBaseInfo", new Class[] { String.class, KylinConfig.class })//
Expand All @@ -460,6 +463,28 @@ public HBaseResponse getHTableInfo(String cubeName, String tableName) throws IOE
return hr;
}

public CubeInstanceResponse createCubeInstanceResponse(CubeInstance cube) {
Preconditions.checkState(!cube.getDescriptor().isDraft());

CubeInstanceResponse r = new CubeInstanceResponse(cube);

CubeDesc cubeDesc = cube.getDescriptor();
DataModelDesc modelDesc = cubeDesc.getModel();
r.setModel(cubeDesc.getModelName());
r.setLastModified(cubeDesc.getLastModified());
r.setPartitionDateStart(cubeDesc.getPartitionDateStart());
// cuz model doesn't have a state the label a model is broken,
// so in some case the model can not be loaded due to some check failed,
// but the cube in this model can still be loaded.
if (modelDesc != null) {
r.setPartitionDateColumn(modelDesc.getPartitionDesc().getPartitionDateColumn());
r.setIs_streaming(modelDesc.getRootFactTable().getTableDesc().getSourceType() == ISourceAware.ID_STREAMING);
}
r.setProject(cube.getProject());

return r;
}

public void updateCubeNotifyList(CubeInstance cube, List<String> notifyList) throws IOException {
aclEvaluate.hasProjectOperationPermission(cube.getProjectInstance());
CubeDesc desc = cube.getDescriptor();
Expand Down
23 changes: 13 additions & 10 deletions server/src/test/java/org/apache/kylin/rest/controller/AccessControllerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,18 @@

package org.apache.kylin.rest.controller;

import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.util.List;

import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.request.AccessRequest;
import org.apache.kylin.rest.response.AccessEntryResponse;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.security.AclEntityType;
import org.apache.kylin.rest.security.AclPermissionType;
import org.apache.kylin.rest.service.AccessService;
Expand All @@ -38,13 +46,6 @@
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;

import java.io.IOException;
import java.util.List;

import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

/**
* @author xduo
*/
Expand Down Expand Up @@ -88,7 +89,8 @@ public void setup() throws Exception {
@Test
public void testBasics() throws IOException {
swichToAdmin();
List<AccessEntryResponse> aes = accessController.getAccessEntities(CUBE_INSTANCE, "a24ca905-1fc6-4f67-985c-38fa5aeafd92");
List<AccessEntryResponse> aes = accessController.getAccessEntities(CUBE_INSTANCE,
"a24ca905-1fc6-4f67-985c-38fa5aeafd92");
Assert.assertTrue(aes.size() == 0);

AccessRequest accessRequest = getAccessRequest(MODELER, ADMINISTRATION);
Expand Down Expand Up @@ -151,7 +153,7 @@ public void testAuthInProjectLevel() throws Exception {
@Test
public void testAuthInCubeLevel() throws Exception {
swichToAdmin();
List<CubeInstance> cubes = cubeController.getCubes(null, null, null, 100000, 0);
List<CubeInstanceResponse> cubes = cubeController.getCubes(null, null, null, 100000, 0);
assertTrue(cubes.size() > 0);
CubeInstance cube = cubes.get(0);
swichToAnalyst();
Expand All @@ -173,7 +175,8 @@ public void testAuthInCubeLevel() throws Exception {
}
swichToAdmin();
List<ProjectInstance> projects = projectController.getProjects(10000, 0);
List<AccessEntryResponse> aes = accessController.grant(PROJECT_INSTANCE, projects.get(0).getUuid(), accessRequest);
List<AccessEntryResponse> aes = accessController.grant(PROJECT_INSTANCE, projects.get(0).getUuid(),
accessRequest);
Assert.assertTrue(aes.size() == 1);
swichToAnalyst();
cubes = cubeController.getCubes(null, null, "default", 100000, 0);
Expand Down
10 changes: 5 additions & 5 deletions server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.kylin.metadata.model.SegmentRange.TSRange;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
import org.apache.kylin.rest.service.ServiceTestBase;
Expand Down Expand Up @@ -120,7 +121,8 @@ public void testBasics() throws IOException {
cubeController.updateNotifyList(newCubeName, notifyList);
cubeController.updateCubeCost(newCubeName, 80);

List<CubeInstance> cubeInstances = cubeController.getCubes(newCubeName, cube.getModelName(), "default", 1, 0);
List<CubeInstanceResponse> cubeInstances = cubeController.getCubes(newCubeName, cube.getModelName(), "default",
1, 0);

CubeInstance cubeInstance = cubeInstances.get(0);
Assert.assertTrue(cubeInstance.getDescriptor().getNotifyList().contains("john@example.com"));
Expand Down Expand Up @@ -170,7 +172,6 @@ public void testDeleteSegmentFromHead() throws IOException {
Assert.assertTrue(segNumber == newSegNumber + 1);
}


@Test
public void testGetHoles() throws IOException {
String cubeName = "test_kylin_cube_with_slr_ready_3_segments";
Expand All @@ -180,7 +181,7 @@ public void testGetHoles() throws IOException {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
List<CubeSegment> segments = cube.getSegments();

final long dateEnd = segments.get(segments.size() -1).getTSRange().end.v;
final long dateEnd = segments.get(segments.size() - 1).getTSRange().end.v;

final long ONEDAY = 24 * 60 * 60000;
cubeService.getCubeManager().appendSegment(cube, new TSRange(dateEnd + ONEDAY, dateEnd + ONEDAY * 2));
Expand All @@ -194,10 +195,9 @@ public void testGetHoles() throws IOException {
Assert.assertTrue(hole.getTSRange().equals(new TSRange(dateEnd, dateEnd + ONEDAY)));
}


@Test
public void testGetCubes() {
List<CubeInstance> cubes = cubeController.getCubes(null, null, null, 1, 0);
List<CubeInstanceResponse> cubes = cubeController.getCubes(null, null, null, 1, 0);
Assert.assertTrue(cubes.size() == 1);
}

Expand Down

0 comments on commit dcd36fa

Please sign in to comment.