KYLIN-2195 Move backward-compatibility properties to common resources

core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java

@@ -18,13 +18,15 @@
 
 package org.apache.kylin.common;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.PrintWriter;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
-import java.util.Set;
+import java.util.Stack;
 
 import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
@@ -35,23 +37,29 @@
 public class BackwardCompatibilityConfig {
 
     private static final Logger logger = LoggerFactory.getLogger(BackwardCompatibilityConfig.class);
-
+
+    private static final String KYLIN_BACKWARD_COMPATIBILITY = "kylin-backward-compatibility";
+
     private final Map<String, String> old2new = Maps.newConcurrentMap();
-
+    private final Map<String, String> old2newPrefix = Maps.newConcurrentMap();
+
     public BackwardCompatibilityConfig() {
-        InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("kylin-backward-compatibility.properties");
-        init(is);
+        ClassLoader loader = Thread.currentThread().getContextClassLoader();
+        init(loader.getResourceAsStream(KYLIN_BACKWARD_COMPATIBILITY + ".properties"));
+        for (int i = 0; i < 10; i++) {
+            init(loader.getResourceAsStream(KYLIN_BACKWARD_COMPATIBILITY + (i) + ".properties"));
+        }
     }
-    
+
     // for test
     BackwardCompatibilityConfig(InputStream is) {
         init(is);
     }
-    
+
     private void init(InputStream is) {
         if (is == null)
             return;
-        
+
         Properties props = new Properties();
         try {
             props.load(is);
@@ -60,50 +68,116 @@ private void init(InputStream is) {
         } finally {
             IOUtils.closeQuietly(is);
         }
-        
+
         for (Entry<Object, Object> kv : props.entrySet()) {
-            old2new.put((String) kv.getKey(), (String) kv.getValue());
+            String key = (String) kv.getKey();
+            String value = (String) kv.getValue();
+            if (value.endsWith("."))
+                old2newPrefix.put(key, value);
+            else
+                old2new.put(key, value);
         }
     }
-    
+
     public String check(String key) {
         String newKey = old2new.get(key);
         if (newKey != null) {
             logger.warn("Config '{}' is deprecated, use '{}' instead", key, newKey);
             return newKey;
-        } else {
-            return key;
         }
+
+        for (String oldPrefix : old2newPrefix.keySet()) {
+            if (key.startsWith(oldPrefix)) {
+                String newPrefix = old2newPrefix.get(oldPrefix);
+                newKey = newPrefix + key.substring(oldPrefix.length() + 1);
+                logger.warn("Config '{}' is deprecated, use '{}' instead", key, newKey);
+                return newKey;
+            }
+        }
+
+        return key;
     }
-    
+
     public Map<String, String> check(Map<String, String> props) {
-        if (containsOldKey(props.keySet()) == false)
-            return props;
-
         LinkedHashMap<String, String> result = new LinkedHashMap<>();
         for (Entry<String, String> kv : props.entrySet()) {
             result.put(check(kv.getKey()), kv.getValue());
         }
         return result;
     }
-    
+
     public Properties check(Properties props) {
-        if (containsOldKey(props.keySet()) == false)
-            return props;
-
         Properties result = new Properties();
         for (Entry<Object, Object> kv : props.entrySet()) {
             result.setProperty(check((String) kv.getKey()), (String) kv.getValue());
         }
         return result;
     }
-
-    private boolean containsOldKey(Set<? extends Object> keySet) {
-        for (Object k : keySet) {
-            if (old2new.containsKey(k))
-                return true;
+
+    // ============================================================================
+
+    public static void main(String[] args) throws IOException {
+        String kylinRepoDir = args.length > 0 ? args[0] : ".";
+        String outputDir = args.length > 1 ? args[1] : ".";
+        generateFindAndReplaceScript(kylinRepoDir, outputDir);
+    }
+
+    private static void generateFindAndReplaceScript(String kylinRepoPath, String outputPath) throws IOException {
+        BackwardCompatibilityConfig bcc = new BackwardCompatibilityConfig();
+        File repoDir = new File(kylinRepoPath).getCanonicalFile();
+        File outputDir = new File(outputPath).getCanonicalFile();
+        PrintWriter out = null;
+
+        // generate sed file
+        File sedFile = new File(outputDir, "upgrade-old-config.sed");
+        try {
+            out = new PrintWriter(sedFile);
+            for (Entry<String, String> e : bcc.old2new.entrySet()) {
+                out.println("s/" + quote(e.getKey()) + "/" + e.getValue() + "/g");
+            }
+            for (Entry<String, String> e : bcc.old2newPrefix.entrySet()) {
+                out.println("s/" + quote(e.getKey()) + "/" + e.getValue() + "/g");
+            }
+        } finally {
+            IOUtils.closeQuietly(out);
         }
-        return false;
+
+        // generate sh file
+        File shFile = new File(outputDir, "upgrade-old-config.sh");
+        try {
+            out = new PrintWriter(shFile);
+            out.println("#!/bin/bash");
+            Stack<File> stack = new Stack<>();
+            stack.push(repoDir);
+            while (!stack.isEmpty()) {
+                File dir = stack.pop();
+                for (File f : dir.listFiles()) {
+                    if (f.getName().startsWith("."))
+                        continue;
+                    if (f.isDirectory())
+                        stack.push(f);
+                    else if (isSourceFile(f))
+                        out.println("sed -i -f upgrade-old-config.sed " + f.getAbsolutePath());
+                }
+            }
+        } finally {
+            IOUtils.closeQuietly(out);
+        }
+
+        System.out.println("Files generated:");
+        System.out.println(shFile);
+        System.out.println(sedFile);
+    }
+
+    private static String quote(String key) {
+        return key.replace(".", "[.]");
     }
 
+    private static boolean isSourceFile(File f) {
+        String name = f.getName();
+        if (name.startsWith(KYLIN_BACKWARD_COMPATIBILITY))
+            return false;
+        else
+            return name.endsWith(".java") || name.endsWith(".js") || name.endsWith(".sh") || name.endsWith(".properties") || name.endsWith(".xml");
+    }
 }

core-common/src/main/resources/kylin-backward-compatibility.properties

@@ -0,0 +1,186 @@
+
+#### ENV ###
+#
+#deploy.env=kylin.env
+#kylin.hdfs.working.dir=kylin.env.hdfs-working-dir
+#
+#
+#### METADATA ###
+#
+#kylin.metadata.url=kylin.metadata.url
+#kylin.realization.providers=kylin.metadata.realization-providers
+#kylin.cube.dimension.customEncodingFactories=kylin.metadata.custom-dimension-encodings
+#kylin.cube.measure.customMeasureType.=kylin.metadata.custom-measure-types.
+#
+#
+#### Dictionary ###
+#
+#kylin.table.snapshot.max_mb=kylin.snapshot.max-mb
+#kylin.snapshot.cache.max.entry=kylin.snapshot.max-cache-entry
+#kylin.dictionary.forest.trie.size.max_mb=kylin.dictionary.forest-trie-max-mb
+#kylin.dict.cache.max.entry=kylin.dictionary.max-cache-entry
+#kylin.dict.growing.enabled=kylin.dictionary.growing-enabled
+#kylin.dict.append.entry.size=kylin.dictionary.append-entry-size
+#kylin.dict.append.cache.size=kylin.dictionary.append-cache-size
+#
+#
+#### CUBE ###
+#
+#kylin.job.cuboid.size.ratio=kylin.cube.size-estimate-ratio
+#kylin.job.cuboid.size.memhungry.ratio=kylin.cube.size-estimate-memhungry-ratio
+#kylin.cube.algorithm=kylin.cube.algorithm
+#kylin.cube.algorithm.auto.threshold=kylin.cube.algorithm.layer-or-inmem-threshold
+#kylin.cube.algorithm.auto.mapper.limit=kylin.cube.algorithm.inmem-split-limit
+#kylin.cube.aggrgroup.max.size=kylin.cube.aggrgroup.max-size
+#kylin.cube.aggrgroup.max.combination=kylin.cube.aggrgroup.max-combination
+#kylin.cube.aggrgroup.isMandatoryOnlyValid=kylin.cube.aggrgroup.is-mandatory-only-valid
+#kylin.cube.building.segment.max=kylin.cube.max-building-segments
+#
+#
+#### JOB ###
+#
+#kylin.job.log.dir=kylin.job.log-dir
+#kylin.job.remote.cli.working.dir=kylin.job.remote-cli-working-dir
+#kylin.job.allow.empty.segment=kylin.job.allow-empty-segment
+#kylin.job.concurrent.max.limit=kylin.job.max-concurrent-jobs
+#kylin.job.cubing.inmem.sampling.percent=kylin.job.sampling-percentage
+#kylin.job.cubing.inmem.sampling.hll.precision=kylin.job.sampling-hll-precision
+#kylin.job.dependency.filterlist=kylin.job.dependency-filter-list
+#kylin.job.retry=kylin.job.retry
+#kylin.job.controller.lock=kylin.job.lock
+#kylin.scheduler.=kylin.job.scheduler.provider.
+#kylin.enable.scheduler=kylin.job.scheduler.default
+#
+#mail.enabled=kylin.job.notification-enabled
+#mail.host=kylin.job.notification-mail-host
+#mail.username=kylin.job.notification-mail-username
+#mail.password=kylin.job.notification-mail-password
+#mail.sender=kylin.job.notification-mail-sender
+#kylin.job.admin.dls=kylin.job.notification-admin-emails
+#
+##for dev
+#kylin.job.run.as.remote.cmd=kylin.job.use-remote-cli
+#kylin.job.remote.cli.port=kylin.job.remote-cli-port
+#kylin.job.remote.cli.hostname=kylin.job.remote-cli-hostname
+#kylin.job.remote.cli.username=kylin.job.remote-cli-username
+#kylin.job.remote.cli.password=kylin.job.remote-cli-password
+#
+#
+#### ENGINE ###
+#
+#kylin.cube.engine.=kylin.engine.provider.
+#kylin.default.cube.engine=kylin.engine.default
+#
+#kylin.job.mr.lib.dir=kylin.engine.mr.lib-dir
+#kylin.job.mr.config.override.=kylin.engine.mr.config-override.
+#kylin.job.jar=kylin.engine.mr.job-jar
+#kylin.job.mapreduce.default.reduce.input.mb=kylin.engine.mr.reduce-input-mb
+#kylin.job.mapreduce.default.reduce.count.ratio=kylin.engine.mr.reduce-count-ratio
+#kylin.job.mapreduce.min.reducer.number=kylin.engine.mr.min-reducer-number
+#kylin.job.mapreduce.max.reducer.number=kylin.engine.mr.max-reducer-number
+#kylin.job.mapreduce.mapper.input.rows=kylin.engine.mr.mapper-input-rows
+#kylin.job.uhc.reducer.count=kylin.engine.mr.uhc-reducer-count
+#kylin.job.yarn.app.rest.check.interval.seconds=kylin.engine.mr.yarn-check-interval-seconds
+#
+#kylin.job.jar.spark=kylin.engine.spark.job-jar
+#kylin.spark.home=kylin.engine.spark.spark-home
+#kylin.spark.master=kylin.engine.spark.spark-master
+#
+##deprecated
+#kylin.job.yarn.app.rest.check.status.url=kylin.engine.mr.yarn-check-status-url
+#
+#
+#### SOURCE ###
+#
+#kylin.source.engine.=kylin.source.provider.
+#kylin.hive.config.override.=kylin.source.hive.config-override.
+#kylin.hive.keep.flat.table=kylin.source.hive.keep-flat-table
+#kylin.job.hive.database.for.intermediatetable=kylin.source.hive.database-for-flat-table
+#kylin.job.hive.intermediatetable.redistribute.enabled=kylin.source.hive.redistribute-flat-table
+#kylin.hive.client=kylin.source.hive.client
+#kylin.hive.beeline.params=kylin.source.hive.beeline-params
+#kylin.hive.create.flat.table.method=kylin.source.hive.create-flat-table-method
+#
+##deprecated
+#hive.url=kylin.source.hive.url
+#hive.user=kylin.source.hive.user
+#hive.password=kylin.source.hive.password
+#hive.table.location.=kylin.source.hive.table-location.
+#
+#
+#### STORAGE ###
+#
+#kylin.storage.url=kylin.storage.url
+#kylin.storage.engine.=kylin.storage.provider.
+#kylin.default.storage.engine=kylin.storage.default
+#
+#kylin.hbase.cluster.fs=kylin.storage.hbase.cluster-fs
+#kylin.hbase.cluster.hdfs.config.file=kylin.storage.hbase.cluster-hdfs-config-file
+#kylin.coprocessor.local.jar=kylin.storage.hbase.coprocessor-local-jar
+#kylin.hbase.region.count.min=kylin.storage.hbase.min-region-count
+#kylin.hbase.region.count.max=kylin.storage.hbase.max-region-count
+#kylin.hbase.hfile.size.gb=kylin.storage.hbase.hfile-size-gb
+#kylin.query.run.local.coprocessor=kylin.storage.hbase.run-local-coprocessor
+#kylin.query.coprocessor.mem.gb=kylin.storage.hbase.coprocessor-mem-gb
+#kylin.query.coprocessor.timeout.seconds=kylin.storage.hbase.coprocessor-timeout-seconds
+#kylin.query.scan.fuzzykey.max=kylin.storage.hbase.max-fuzzykey-scan
+#kylin.query.storage.visit.scanrange.max=kylin.storage.hbase.max-visit-scanrange
+#kylin.query.storage.default.gtstorage=kylin.storage.hbase.gtstorage
+#kylin.hbase.scan.cache_rows=kylin.storage.hbase.scan-cache-rows
+#kylin.hbase.region.cut=kylin.storage.hbase.region-cut-gb
+#kylin.hbase.scan.max_result_size=kylin.storage.hbase.max-scan-result-bytes
+#kylin.hbase.default.compression.codec=kylin.storage.hbase.compression-codec
+#kylin.hbase.default.encoding=kylin.storage.hbase.rowkey-encoding
+#kylin.hbase.default.block.size=kylin.storage.hbase.block-size-bytes
+#kylin.hbase.small.family.block.size=kylin.storage.hbase.small-family-block-size-bytes
+#kylin.owner=kylin.storage.hbase.owner-tag
+#kylin.query.endpoint.compression.result=kylin.storage.hbase.endpoint-compress-result
+#kylin.query.hbase.hconnection.threads.max=kylin.storage.hbase.max-hconnection-threads
+#kylin.query.hbase.hconnection.threads.core=kylin.storage.hbase.core-hconnection-threads
+#kylin.query.hbase.hconnection.threads.alive.seconds=kylin.storage.hbase.hconnection-threads-alive-seconds
+#
+#
+#### QUERY ###
+#
+#kylin.query.pushdown.limit.max=kylin.query.max-limit-pushdown
+#kylin.query.scan.threshold=kylin.query.scan-threshold
+#kylin.query.filter.derived_in.max=kylin.query.derived-filter-translation-threshold
+#kylin.query.badquery.stacktrace.depth=kylin.query.badquery-stacktrace-depth
+#kylin.query.badquery.history.num=kylin.query.badquery-history-number
+#kylin.query.badquery.alerting.seconds=kylin.query.badquery-alerting-seconds
+#kylin.query.badquery.detect.interval.seconds=kylin.query.badquery-detect-interval
+#kylin.query.badquery.persistent.enable=kylin.query.badquery-persistent-enabled
+#kylin.query.transformers=kylin.query.transformers
+#kylin.query.cache.enabled=kylin.query.cache-enabled
+#kylin.query.cache.threshold.duration=kylin.query.cache-threshold-duration
+#kylin.query.cache.threshold.scancount=kylin.query.cache-threshold-scan-count
+#kylin.query.mem.budget=kylin.query.memory-budget
+#kylin.query.ignore_unknown_function=kylin.query.ignore-unknown-function
+#kylin.query.dim.distinct.max=kylin.query.max-dimension-count-distinct
+#kylin.query.security.enabled=kylin.query.security-enabled
+#kylin.query.access.controller=kylin.query.access-controller
+#kylin.query.udf.=kylin.query.udf.
+#
+#
+#### SERVER ###
+#
+#kylin.init.tasks=kylin.server.init-tasks
+#kylin.server.mode=kylin.server.mode
+#kylin.cluster.name=kylin.server.cluster-name
+#kylin.rest.servers=kylin.server.cluster-servers
+#kylin.rest.workers.per.server=kylin.server.sequence-sql.workers-per-server
+#kylin.query.sequence.expire.time=kylin.server.sequence-sql.expire-time
+#kylin.query.metrics.enabled=kylin.server.query-metrics-enabled
+#kylin.query.metrics.percentiles.intervals=kylin.server.query-metrics-percentiles-intervals
+#
+#
+#### WEB ###
+#
+#kylin.rest.timezone=kylin.web.timezone
+#crossdomain.enable=kylin.web.cross-domain-enabled
+
+
+### TEST ###
+
+kylin.test.bcc.old.key=kylin.test.bcc.new.key
+