From f12088637347385ba1d81eb345c39d2276c2bbe3 Mon Sep 17 00:00:00 2001 From: Wang Ken Date: Mon, 25 Sep 2017 13:59:46 +0800 Subject: [PATCH] KYLIN-2894 Query cache expiration strategy switches from manual invalidation to signature checking --- .../apache/kylin/common/KylinConfigBase.java | 5 + .../kylin/rest/response/SQLResponse.java | 14 +- .../kylin/rest/service/CacheService.java | 4 +- .../kylin/rest/service/QueryService.java | 100 +++++++---- .../rest/signature/ComponentSignature.java | 31 ++++ .../signature/RealizationSetCalculator.java | 101 +++++++++++ .../rest/signature/RealizationSignature.java | 164 ++++++++++++++++++ .../rest/signature/SegmentSignature.java | 65 +++++++ .../rest/signature/SignatureCalculator.java | 28 +++ .../rest/util/SQLResponseSignatureUtil.java | 68 ++++++++ .../org/apache/kylin/rest/bean/BeanTest.java | 2 +- .../rest/controller/QueryControllerTest.java | 2 +- 12 files changed, 545 insertions(+), 39 deletions(-) create mode 100644 server-base/src/main/java/org/apache/kylin/rest/signature/ComponentSignature.java create mode 100644 server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSetCalculator.java create mode 100644 server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSignature.java create mode 100644 server-base/src/main/java/org/apache/kylin/rest/signature/SegmentSignature.java create mode 100644 server-base/src/main/java/org/apache/kylin/rest/signature/SignatureCalculator.java create mode 100644 server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 5577307ac32..135d6e628b1 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1553,6 +1553,11 @@ public String getQueryRealizationFilter() { return getOptional("kylin.query.realization-filter", null); } + public String getSQLResponseSignatureClass() { + return this.getOptional("kylin.query.signature-class", + "org.apache.kylin.rest.signature.RealizationSetCalculator"); + } + // ============================================================================ // SERVER // ============================================================================ diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java index 0bdf037230c..05027983493 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java +++ b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java @@ -76,6 +76,9 @@ public class SQLResponse implements Serializable { protected String traceUrl = null; + // it's sql response signature for cache checking, no need to return and should be JsonIgnore + protected String signature; + public SQLResponse() { } @@ -205,7 +208,16 @@ public String getTraceUrl() { public void setTraceUrl(String traceUrl) { this.traceUrl = traceUrl; } - + + @JsonIgnore + public String getSignature() { + return signature; + } + + public void setSignature(String signature) { + this.signature = signature; + } + @JsonIgnore public List getCubeSegmentStatisticsList() { try { diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java index 10ab90b82b9..67d49d9f522 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java @@ -117,8 +117,8 @@ public void notifyMetadataChange(String entity, Event event, String cacheKey) th public void cleanDataCache(String project) { if (cacheManager != null) { - logger.info("cleaning cache for project " + project + " (currently remove all entries)"); - cacheManager.getCache(QueryService.SUCCESS_QUERY_CACHE).removeAll(); + logger.info("cleaning cache for project " + project + " (currently remove exception entries)"); + // cacheManager.getCache(QueryService.SUCCESS_QUERY_CACHE).removeAll(); cacheManager.getCache(QueryService.EXCEPTION_QUERY_CACHE).removeAll(); } else { logger.warn("skip cleaning cache for project " + project); diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java index 82624723dd2..fb13ff57623 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -110,6 +110,7 @@ import org.apache.kylin.rest.util.AclEvaluate; import org.apache.kylin.rest.util.AclPermissionUtil; import org.apache.kylin.rest.util.QueryRequestLimits; +import org.apache.kylin.rest.util.SQLResponseSignatureUtil; import org.apache.kylin.rest.util.TableauInterceptor; import org.apache.kylin.storage.hybrid.HybridInstance; import org.apache.kylin.storage.hybrid.HybridManager; @@ -117,18 +118,17 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.cache.Cache; +import org.springframework.cache.CacheManager; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.Lists; -import net.sf.ehcache.Cache; -import net.sf.ehcache.CacheManager; -import net.sf.ehcache.Element; - /** * @author xduo */ @@ -226,7 +226,7 @@ public SQLResponse update(SQLRequest sqlRequest) throws Exception { columnMetas.add(new SelectedColumnMeta(false, false, false, false, 1, false, Integer.MAX_VALUE, "c0", "c0", null, null, null, Integer.MAX_VALUE, 128, 1, "char", false, false, false)); - return buildSqlResponse(true, r.getFirst(), columnMetas); + return buildSqlResponse(sqlRequest.getProject(), true, r.getFirst(), columnMetas); } catch (Exception e) { logger.info("pushdown engine failed to finish current non-select query"); @@ -313,6 +313,12 @@ public void logQuery(final String queryId, final SQLRequest request, final SQLRe } } + if (realizationNames.isEmpty()) { + if (!Strings.isNullOrEmpty(response.getCube())) { + realizationNames.addAll(Lists.newArrayList(response.getCube().split(","))); + } + } + int resultRowCount = 0; if (!response.getIsException() && response.getResults() != null) { resultRowCount = response.getResults().size(); @@ -458,6 +464,8 @@ private SQLResponse queryAndUpdateCache(SQLRequest sqlRequest, boolean queryCach String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount())); if (checkCondition(queryCacheEnabled, "query cache is disabled") // + && checkCondition(!Strings.isNullOrEmpty(sqlResponse.getCube()), + "query does not hit cube nor hybrid") // && checkCondition(!sqlResponse.getIsException(), "query has exception") // && checkCondition( !(sqlResponse.isPushDown() @@ -473,7 +481,7 @@ && checkCondition( && checkCondition(sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold(), "query response is too large: {} ({})", sqlResponse.getResults().size(), kylinConfig.getLargeQueryThreshold())) { - cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest.getCacheKey(), sqlResponse)); + cacheManager.getCache(SUCCESS_QUERY_CACHE).put(sqlRequest.getCacheKey(), sqlResponse); } } catch (Throwable e) { // calcite may throw AssertError @@ -482,15 +490,13 @@ && checkCondition(sqlResponse.getResults().size() < kylinConfig.getLargeQueryThr logger.error("Exception while executing query", e); String errMsg = makeErrorMsgUserFriendly(e); - sqlResponse = new SQLResponse(null, null, null, 0, true, errMsg, false, false); - sqlResponse.setTotalScanCount(queryContext.getScannedRows()); - sqlResponse.setTotalScanBytes(queryContext.getScannedBytes()); - sqlResponse.setCubeSegmentStatisticsList(queryContext.getCubeSegmentStatisticsResultList()); + sqlResponse = buildSqlResponse(sqlRequest.getProject(), false, null, null, true, errMsg); + sqlResponse.setThrowable(e.getCause() == null ? e : ExceptionUtils.getRootCause(e)); if (queryCacheEnabled && e.getCause() != null && ExceptionUtils.getRootCause(e) instanceof ResourceLimitExceededException) { Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE); - exceptionCache.put(new Element(sqlRequest.getCacheKey(), sqlResponse)); + exceptionCache.put(sqlRequest.getCacheKey(), sqlResponse); } } return sqlResponse; @@ -515,22 +521,36 @@ private String getUserName() { } public SQLResponse searchQueryInCache(SQLRequest sqlRequest) { - SQLResponse response = null; - Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE); - Cache successCache = cacheManager.getCache(SUCCESS_QUERY_CACHE); - - Element element = null; - if ((element = exceptionCache.get(sqlRequest.getCacheKey())) != null) { - logger.info("The sqlResponse is found in EXCEPTION_QUERY_CACHE"); - response = (SQLResponse) element.getObjectValue(); - response.setHitExceptionCache(true); - } else if ((element = successCache.get(sqlRequest.getCacheKey())) != null) { - logger.info("The sqlResponse is found in SUCCESS_QUERY_CACHE"); - response = (SQLResponse) element.getObjectValue(); - response.setStorageCacheUsed(true); + String[] cacheTypes = new String[] { EXCEPTION_QUERY_CACHE, SUCCESS_QUERY_CACHE }; + for (String cacheType : cacheTypes) { + Cache cache = cacheManager.getCache(cacheType); + Cache.ValueWrapper wrapper = cache.get(sqlRequest.getCacheKey()); + if (wrapper == null) { + continue; + } + SQLResponse response = (SQLResponse) wrapper.get(); + if (response == null) { + return null; + } + logger.info("The sqlResponse is found in " + cacheType); + if (!SQLResponseSignatureUtil.checkSignature(getConfig(), response, sqlRequest.getProject())) { + logger.info("The sql response signature is changed. Remove it from QUERY_CACHE."); + cache.evict(sqlRequest.getCacheKey()); + return null; + } else { + switch (cacheType) { + case EXCEPTION_QUERY_CACHE: + response.setHitExceptionCache(true); + break; + case SUCCESS_QUERY_CACHE: + response.setStorageCacheUsed(true); + break; + default: + } + } + return response; } - - return response; + return null; } private SQLResponse queryWithSqlMassage(SQLRequest sqlRequest) throws Exception { @@ -579,7 +599,8 @@ private SQLResponse queryWithSqlMassage(SQLRequest sqlRequest) throws Exception List> results = Lists.newArrayList(); List columnMetas = Lists.newArrayList(); if (BackdoorToggles.getPrepareOnly()) { - return getPrepareOnlySqlResponse(correctedSql, conn, false, results, columnMetas); + return getPrepareOnlySqlResponse(sqlRequest.getProject(), correctedSql, conn, false, results, + columnMetas); } if (!isPrepareRequest) { return executeRequest(correctedSql, sqlRequest, conn); @@ -893,7 +914,7 @@ private SQLResponse executeRequest(String correctedSql, SQLRequest sqlRequest, C close(resultSet, stat, null); //conn is passed in, not my duty to close } - return buildSqlResponse(isPushDown, r.getFirst(), r.getSecond()); + return buildSqlResponse(sqlRequest.getProject(), isPushDown, r.getFirst(), r.getSecond()); } private SQLResponse executePrepareRequest(String correctedSql, PrepareSqlRequest sqlRequest, @@ -921,7 +942,7 @@ private SQLResponse executePrepareRequest(String correctedSql, PrepareSqlRequest DBUtils.closeQuietly(resultSet); } - return buildSqlResponse(isPushDown, r.getFirst(), r.getSecond()); + return buildSqlResponse(sqlRequest.getProject(), isPushDown, r.getFirst(), r.getSecond()); } private Pair>, List> pushDownQuery(SQLRequest sqlRequest, String correctedSql, @@ -972,7 +993,8 @@ protected String makeErrorMsgUserFriendly(Throwable e) { return QueryUtil.makeErrorMsgUserFriendly(e); } - private SQLResponse getPrepareOnlySqlResponse(String correctedSql, Connection conn, Boolean isPushDown, + private SQLResponse getPrepareOnlySqlResponse(String projectName, String correctedSql, Connection conn, + Boolean isPushDown, List> results, List columnMetas) throws SQLException { CalcitePrepareImpl.KYLIN_ONLY_PREPARE.set(true); @@ -1018,7 +1040,7 @@ private SQLResponse getPrepareOnlySqlResponse(String correctedSql, Connection co DBUtils.closeQuietly(preparedStatement); } - return buildSqlResponse(isPushDown, results, columnMetas); + return buildSqlResponse(projectName, isPushDown, results, columnMetas); } private boolean isPrepareStatementWithParams(SQLRequest sqlRequest) { @@ -1028,10 +1050,17 @@ private boolean isPrepareStatementWithParams(SQLRequest sqlRequest) { return false; } - private SQLResponse buildSqlResponse(Boolean isPushDown, List> results, + private SQLResponse buildSqlResponse(String projectName, Boolean isPushDown, List> results, List columnMetas) { + return buildSqlResponse(projectName, isPushDown, results, columnMetas, false, null); + } + + private SQLResponse buildSqlResponse(String projectName, Boolean isPushDown, List> results, + List columnMetas, boolean isException, String exceptionMessage) { boolean isPartialResult = false; + + List realizations = Lists.newLinkedList(); StringBuilder cubeSb = new StringBuilder(); StringBuilder logSb = new StringBuilder("Processed rows for each storageContext: "); QueryContext queryContext = QueryContextFacade.current(); @@ -1049,17 +1078,20 @@ private SQLResponse buildSqlResponse(Boolean isPushDown, List> resu realizationName = ctx.realization.getName(); realizationType = ctx.realization.getStorageType(); + + realizations.add(realizationName); } queryContext.setContextRealization(ctx.id, realizationName, realizationType); } } logger.info(logSb.toString()); - SQLResponse response = new SQLResponse(columnMetas, results, cubeSb.toString(), 0, false, null, isPartialResult, - isPushDown); + SQLResponse response = new SQLResponse(columnMetas, results, cubeSb.toString(), 0, isException, + exceptionMessage, isPartialResult, isPushDown); response.setTotalScanCount(queryContext.getScannedRows()); response.setTotalScanBytes(queryContext.getScannedBytes()); response.setCubeSegmentStatisticsList(queryContext.getCubeSegmentStatisticsResultList()); + response.setSignature(SQLResponseSignatureUtil.createSignature(getConfig(), response, projectName)); return response; } diff --git a/server-base/src/main/java/org/apache/kylin/rest/signature/ComponentSignature.java b/server-base/src/main/java/org/apache/kylin/rest/signature/ComponentSignature.java new file mode 100644 index 00000000000..7b3a2e4b9f1 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/signature/ComponentSignature.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.signature; + +import java.io.Serializable; + +abstract class ComponentSignature implements Serializable, Comparable { + + public abstract String getKey(); + + @Override + public int compareTo(T o) { + return getKey().compareTo(o.getKey()); + } +} diff --git a/server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSetCalculator.java b/server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSetCalculator.java new file mode 100644 index 00000000000..63139f38f61 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSetCalculator.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.signature; + +import java.security.MessageDigest; +import java.util.Set; + +import org.apache.commons.codec.binary.Base64; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.Pair; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.rest.response.SQLResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.CharMatcher; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; + +public class RealizationSetCalculator implements SignatureCalculator { + + public static final Logger logger = LoggerFactory.getLogger(RealizationSetCalculator.class); + + @Override + public String calculateSignature(KylinConfig config, SQLResponse sqlResponse, ProjectInstance project) { + Set realizations = getRealizations(config, sqlResponse.getCube(), project); + if (realizations == null) { + return null; + } + Set signatureSet = Sets.newTreeSet(); + for (String realization : realizations) { + RealizationSignature realizationSignature = getRealizationSignature(config, realization); + if (realizationSignature != null) { + signatureSet.add(realizationSignature); + } + } + if (signatureSet.isEmpty()) { + return null; + } + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] signature = md.digest(signatureSet.toString().getBytes("UTF-8")); + return new String(Base64.encodeBase64(signature), "UTF-8"); + } catch (Exception e) { + logger.warn("Failed to calculate signature due to " + e); + return null; + } + } + + protected Set getRealizations(KylinConfig config, String cubes, ProjectInstance project) { + if (Strings.isNullOrEmpty(cubes)) { + return null; + } + String[] realizations = parseNamesFromCanonicalNames(cubes.split(",")); + return Sets.newHashSet(realizations); + } + + protected static RealizationSignature getRealizationSignature(KylinConfig config, String realizationName) { + RealizationSignature result = RealizationSignature.HybridSignature.getHybridSignature(config, realizationName); + if (result == null) { + result = RealizationSignature.CubeSignature.getCubeSignature(config, realizationName); + } + return result; + } + + private static String[] parseNamesFromCanonicalNames(String[] canonicalNames) { + String[] result = new String[canonicalNames.length]; + for (int i = 0; i < canonicalNames.length; i++) { + result[i] = parseCanonicalName(canonicalNames[i]).getSecond(); + } + return result; + } + + /** + * @param canonicalName + * @return type and name pair for realization + */ + private static Pair parseCanonicalName(String canonicalName) { + Iterable parts = Splitter.on(CharMatcher.anyOf("[]=,")).split(canonicalName); + String[] partsStr = Iterables.toArray(parts, String.class); + return new Pair<>(partsStr[0], partsStr[2]); + } +} diff --git a/server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSignature.java b/server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSignature.java new file mode 100644 index 00000000000..9e540852505 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSignature.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.signature; + +import java.util.List; +import java.util.Set; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.CubeSegment; +import org.apache.kylin.metadata.model.SegmentStatusEnum; +import org.apache.kylin.metadata.realization.IRealization; +import org.apache.kylin.metadata.realization.RealizationStatusEnum; +import org.apache.kylin.metadata.realization.RealizationType; +import org.apache.kylin.storage.hybrid.HybridInstance; +import org.apache.kylin.storage.hybrid.HybridManager; + +import com.google.common.collect.Sets; + +public abstract class RealizationSignature extends ComponentSignature { + + static class CubeSignature extends RealizationSignature { + public final String name; + public final RealizationStatusEnum status; + public final Set segmentSignatureSet; + + private CubeSignature(String name, RealizationStatusEnum status, Set segmentSignatureSet) { + this.name = name; + this.status = status; + this.segmentSignatureSet = segmentSignatureSet; + } + + public String getKey() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + CubeSignature that = (CubeSignature) o; + + if (name != null ? !name.equals(that.name) : that.name != null) + return false; + if (status != that.status) + return false; + return segmentSignatureSet != null ? segmentSignatureSet.equals(that.segmentSignatureSet) + : that.segmentSignatureSet == null; + } + + @Override + public int hashCode() { + int result = name != null ? name.hashCode() : 0; + result = 31 * result + (status != null ? status.hashCode() : 0); + result = 31 * result + (segmentSignatureSet != null ? segmentSignatureSet.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return name + "-" + status + ":" + + (segmentSignatureSet != null ? Sets.newTreeSet(segmentSignatureSet) : null); + } + + static CubeSignature getCubeSignature(KylinConfig config, String realizationName) { + CubeInstance cubeInstance = CubeManager.getInstance(config).getCube(realizationName); + if (cubeInstance == null) { + return null; + } + if (!cubeInstance.isReady()) { + return new CubeSignature(realizationName, RealizationStatusEnum.DISABLED, null); + } + List readySegments = cubeInstance.getSegments(SegmentStatusEnum.READY); + Set segmentSignatureSet = Sets.newHashSetWithExpectedSize(readySegments.size()); + for (CubeSegment cubeSeg : readySegments) { + segmentSignatureSet.add(new SegmentSignature(cubeSeg.getName(), cubeSeg.getLastBuildTime())); + } + return new CubeSignature(realizationName, RealizationStatusEnum.READY, segmentSignatureSet); + } + } + + static class HybridSignature extends RealizationSignature { + public final String name; + public final Set realizationSignatureSet; + + private HybridSignature(String name, Set realizationSignatureSet) { + this.name = name; + this.realizationSignatureSet = realizationSignatureSet; + } + + public String getKey() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + HybridSignature that = (HybridSignature) o; + + if (name != null ? !name.equals(that.name) : that.name != null) + return false; + return realizationSignatureSet != null ? realizationSignatureSet.equals(that.realizationSignatureSet) + : that.realizationSignatureSet == null; + + } + + @Override + public int hashCode() { + int result = name != null ? name.hashCode() : 0; + result = 31 * result + (realizationSignatureSet != null ? realizationSignatureSet.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return name + ":" + (realizationSignatureSet != null ? Sets.newTreeSet(realizationSignatureSet) : null); + } + + static HybridSignature getHybridSignature(KylinConfig config, String realizationName) { + HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(realizationName); + if (hybridInstance == null) { + return null; + } + IRealization[] realizations = hybridInstance.getRealizations(); + Set realizationSignatureSet = Sets.newHashSetWithExpectedSize(realizations.length); + for (IRealization realization : realizations) { + RealizationSignature realizationSignature = null; + if (realization.getType() == RealizationType.CUBE) { + realizationSignature = CubeSignature.getCubeSignature(config, realization.getName()); + } else if (realization.getType() == RealizationType.HYBRID) { + realizationSignature = getHybridSignature(config, realization.getName()); + } + if (realizationSignature != null) { + realizationSignatureSet.add(realizationSignature); + } + } + return new HybridSignature(realizationName, realizationSignatureSet); + } + } +} diff --git a/server-base/src/main/java/org/apache/kylin/rest/signature/SegmentSignature.java b/server-base/src/main/java/org/apache/kylin/rest/signature/SegmentSignature.java new file mode 100644 index 00000000000..800dd99b46a --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/signature/SegmentSignature.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.signature; + +class SegmentSignature extends ComponentSignature { + public final String name; + public final long lastBuildTime; + + public SegmentSignature(String name, long lastBuildTime) { + this.name = name; + this.lastBuildTime = lastBuildTime; + } + + public String getKey() { + return name; + } + + @Override + public int compareTo(SegmentSignature o) { + return name.compareTo(o.name); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + SegmentSignature that = (SegmentSignature) o; + + if (lastBuildTime != that.lastBuildTime) + return false; + return name != null ? name.equals(that.name) : that.name == null; + + } + + @Override + public int hashCode() { + int result = name != null ? name.hashCode() : 0; + result = 31 * result + (int) (lastBuildTime ^ (lastBuildTime >>> 32)); + return result; + } + + @Override + public String toString() { + return name + ":" + lastBuildTime; + } +} diff --git a/server-base/src/main/java/org/apache/kylin/rest/signature/SignatureCalculator.java b/server-base/src/main/java/org/apache/kylin/rest/signature/SignatureCalculator.java new file mode 100644 index 00000000000..1f94bebbb7a --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/signature/SignatureCalculator.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.signature; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.rest.response.SQLResponse; + +public interface SignatureCalculator { + + String calculateSignature(KylinConfig config, SQLResponse sqlResponse, ProjectInstance project); +} diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java new file mode 100644 index 00000000000..c6d35073785 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.util; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.metadata.project.ProjectManager; +import org.apache.kylin.rest.response.SQLResponse; +import org.apache.kylin.rest.signature.RealizationSetCalculator; +import org.apache.kylin.rest.signature.SignatureCalculator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; + +public class SQLResponseSignatureUtil { + + public static final Logger logger = LoggerFactory.getLogger(SQLResponseSignatureUtil.class); + + public static boolean checkSignature(KylinConfig config, SQLResponse sqlResponse, String projectName) { + String old = sqlResponse.getSignature(); + if (old == null) { + return false; + } + String latest = createSignature(config, sqlResponse, projectName); + return old.equals(latest); + } + + public static String createSignature(KylinConfig config, SQLResponse sqlResponse, String projectName) { + ProjectInstance project = ProjectManager.getInstance(config).getProject(projectName); + Preconditions.checkNotNull(project); + + SignatureCalculator signatureCalculator; + try { + Class signatureClass = getSignatureClass(project.getConfig()); + signatureCalculator = (SignatureCalculator) signatureClass.getConstructor().newInstance(); + } catch (Exception e) { + logger.warn("Will use default signature since fail to construct signature due to " + e); + signatureCalculator = new RealizationSetCalculator(); + } + return signatureCalculator.calculateSignature(config, sqlResponse, project); + } + + private static Class getSignatureClass(KylinConfig config) { + try { + return Class.forName(config.getSQLResponseSignatureClass()); + } catch (ClassNotFoundException e) { + logger.warn("Will use default signature since cannot find class " + config.getSQLResponseSignatureClass()); + return RealizationSetCalculator.class; + } + } +} diff --git a/server-base/src/test/java/org/apache/kylin/rest/bean/BeanTest.java b/server-base/src/test/java/org/apache/kylin/rest/bean/BeanTest.java index e1a1228c54a..09191e0cddc 100644 --- a/server-base/src/test/java/org/apache/kylin/rest/bean/BeanTest.java +++ b/server-base/src/test/java/org/apache/kylin/rest/bean/BeanTest.java @@ -54,7 +54,7 @@ public void test() { } catch (IntrospectionException e) { } - new SQLResponse(null, null, null, 0, true, null, false, false); + new SQLResponse(null, null, 0, true, null); SelectedColumnMeta coulmnMeta = new SelectedColumnMeta(false, false, false, false, 0, false, 0, null, null, null, null, null, 0, 0, 0, null, false, false, false); diff --git a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java index 7c5f253a2f9..2225096da54 100644 --- a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java +++ b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java @@ -31,7 +31,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import net.sf.ehcache.CacheManager; +import org.springframework.cache.CacheManager; /** * @author xduo