diff --git a/.github/workflows/scripts/.pinot_linter.sh b/.github/workflows/scripts/.pinot_linter.sh index ca74747ee4b..ef4ba7a5f3d 100755 --- a/.github/workflows/scripts/.pinot_linter.sh +++ b/.github/workflows/scripts/.pinot_linter.sh @@ -26,8 +26,8 @@ ifconfig netstat -i -mvn -B -ntp license:check || exit 1 -mvn -B -ntp checkstyle:check || exit 1 -mvn -B -ntp spotless:check || exit 1 -mvn -B -ntp enforcer:enforce || exit 1 +mvn -B -ntp -T1C license:check || exit 1 +mvn -B -ntp -T1C checkstyle:check || exit 1 +mvn -B -ntp -T1C spotless:check || exit 1 +mvn -B -ntp -T1C enforcer:enforce || exit 1 diff --git a/contrib/pinot-fmpp-maven-plugin/pom.xml b/contrib/pinot-fmpp-maven-plugin/pom.xml deleted file mode 100644 index 0fbdc44f48a..00000000000 --- a/contrib/pinot-fmpp-maven-plugin/pom.xml +++ /dev/null @@ -1,127 +0,0 @@ - - - - 4.0.0 - - pinot - org.apache.pinot - 1.2.0-SNAPSHOT - ../.. - - - pinot-fmpp-maven-plugin - Pinot FMPP plugin - https://pinot.apache.org/ - maven-plugin - - ${basedir}/../.. - 3.8.2 - 0.9.16 - 2.3.32 - - - - - commons-io - commons-io - - - org.apache.maven - maven-core - ${maven.version} - - - org.codehaus.plexus - plexus-utils - - - org.eclipse.sisu - org.eclipse.sisu.plexus - - - - - org.apache.maven - maven-plugin-api - ${maven.version} - - - org.eclipse.sisu - org.eclipse.sisu.plexus - - - - - net.sourceforge.fmpp - fmpp - ${fmpp.version} - - - org.freemarker - freemarker - - - - - org.freemarker - freemarker - ${freemarker.version} - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - true - - - - org.apache.maven.plugins - maven-plugin-plugin - - pinot-fmpp - - - - default-descriptor - - descriptor - - process-classes - - - help-descriptor - - helpmojo - - process-classes - - - - - - diff --git a/contrib/pinot-fmpp-maven-plugin/src/main/java/org/apache/pinot/fmpp/FMPPMojo.java b/contrib/pinot-fmpp-maven-plugin/src/main/java/org/apache/pinot/fmpp/FMPPMojo.java deleted file mode 100644 index 787ac7606cd..00000000000 --- a/contrib/pinot-fmpp-maven-plugin/src/main/java/org/apache/pinot/fmpp/FMPPMojo.java +++ /dev/null @@ -1,270 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pinot.fmpp; - -import com.google.common.base.Joiner; -import com.google.common.base.Stopwatch; -import fmpp.Engine; -import fmpp.ProgressListener; -import fmpp.progresslisteners.TerseConsoleProgressListener; -import fmpp.setting.Settings; -import fmpp.util.MiscUtil; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.commons.io.FileUtils; -import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.plugin.MojoFailureException; -import org.apache.maven.project.MavenProject; - -import static java.lang.String.format; - - -/** - * a maven plugin to run the freemarker generation incrementally - * (if output has not changed, the files are not touched) - * - * @goal generate - * @phase generate-sources - */ -public class FMPPMojo extends AbstractMojo { - - /** - * Used to add new source directories to the build. - * - * @parameter default-value="${project}" - * @required - * @readonly - **/ - private MavenProject project; - - /** - * Where to find the FreeMarker template files. - * - * @parameter default-value="src/main/resources/fmpp/templates/" - * @required - */ - private File templates; - - /** - * Where to write the generated files of the output files. - * - * @parameter default-value="${project.build.directory}/generated-sources/fmpp/" - * @required - */ - private File output; - - /** - * Location of the FreeMarker config file. - * - * @parameter default-value="src/main/resources/fmpp/config.fmpp" - * @required - */ - private File config; - - /** - * compilation scope to be added to ("compile" or "test") - * - * @parameter default-value="compile" - * @required - */ - private String scope; - - /** - * FMPP data model build parameter. - * - * @see FMPP Data Model Building - * @parameter default-value="" - */ - private String data; - - /** - * if maven properties are added as data - * - * @parameter default-value="true" - * @required - */ - private boolean addMavenDataLoader; - - @Override - public void execute() - throws MojoExecutionException, MojoFailureException { - if (project == null) { - throw new MojoExecutionException("This plugin can only be used inside a project."); - } - String outputPath = output.getAbsolutePath(); - if ((!output.exists() && !output.mkdirs()) || !output.isDirectory()) { - throw new MojoFailureException("can not write to output dir: " + outputPath); - } - String templatesPath = templates.getAbsolutePath(); - if (!templates.exists() || !templates.isDirectory()) { - throw new MojoFailureException("templates not found in dir: " + outputPath); - } - - // add the output directory path to the project source directories - switch (scope) { - case "compile": - project.addCompileSourceRoot(outputPath); - break; - case "test": - project.addTestCompileSourceRoot(outputPath); - break; - default: - throw new MojoFailureException("scope must be compile or test"); - } - - final Stopwatch sw = Stopwatch.createStarted(); - try { - getLog().info( - format("Freemarker generation:\n scope: %s,\n config: %s,\n templates: %s", scope, config.getAbsolutePath(), - templatesPath)); - final File tmp = Files.createTempDirectory("freemarker-tmp").toFile(); - String tmpPath = tmp.getAbsolutePath(); - final String tmpPathNormalized = tmpPath.endsWith(File.separator) ? tmpPath : tmpPath + File.separator; - Settings settings = new Settings(new File(".")); - settings.set(Settings.NAME_SOURCE_ROOT, templatesPath); - settings.set(Settings.NAME_OUTPUT_ROOT, tmp.getAbsolutePath()); - settings.load(config); - settings.addProgressListener(new TerseConsoleProgressListener()); - settings.addProgressListener(new ProgressListener() { - @Override - public void notifyProgressEvent(Engine engine, int event, File src, int pMode, Throwable error, Object param) - throws Exception { - if (event == EVENT_END_PROCESSING_SESSION) { - getLog().info(format("Freemarker generation took %dms", sw.elapsed(TimeUnit.MILLISECONDS))); - sw.reset(); - Report report = moveIfChanged(tmp, tmpPathNormalized); - if (!tmp.delete()) { - throw new MojoFailureException(format("can not delete %s", tmp)); - } - getLog().info(format("Incremental output update took %dms", sw.elapsed(TimeUnit.MILLISECONDS))); - getLog().info(format("new: %d", report.newFiles)); - getLog().info(format("changed: %d", report.changedFiles)); - getLog().info(format("unchanged: %d", report.unchangedFiles)); - } - } - }); - List dataValues = new ArrayList<>(); - if (addMavenDataLoader) { - getLog().info("Adding maven data loader"); - settings.setEngineAttribute(MavenDataLoader.MAVEN_DATA_ATTRIBUTE, new MavenDataLoader.MavenData(project)); - dataValues.add(format("maven: %s()", MavenDataLoader.class.getName())); - } - if (data != null) { - dataValues.add(data); - } - if (!dataValues.isEmpty()) { - String dataString = Joiner.on(",").join(dataValues); - getLog().info("Setting data loader " + dataString); - - settings.add(Settings.NAME_DATA, dataString); - } - settings.execute(); - } catch (Exception e) { - throw new MojoFailureException(MiscUtil.causeMessages(e), e); - } - } - - private static final class Report { - int changedFiles; - int unchangedFiles; - int newFiles; - - Report(int changedFiles, int unchangedFiles, int newFiles) { - super(); - this.changedFiles = changedFiles; - this.unchangedFiles = unchangedFiles; - this.newFiles = newFiles; - } - - public Report() { - this(0, 0, 0); - } - - void add(Report other) { - changedFiles += other.changedFiles; - unchangedFiles += other.unchangedFiles; - newFiles += other.newFiles; - } - - public void addChanged() { - ++changedFiles; - } - - public void addNew() { - ++newFiles; - } - - public void addUnchanged() { - ++unchangedFiles; - } - } - - private Report moveIfChanged(File root, String tmpPath) - throws MojoFailureException, IOException { - Report report = new Report(); - for (File file : root.listFiles()) { - if (file.isDirectory()) { - report.add(moveIfChanged(file, tmpPath)); - if (!file.delete()) { - throw new MojoFailureException(format("can not delete %s", file)); - } - } else { - String absPath = file.getAbsolutePath(); - if (!absPath.startsWith(tmpPath)) { - throw new MojoFailureException(format("%s should start with %s", absPath, tmpPath)); - } - String relPath = absPath.substring(tmpPath.length()); - File outputFile = new File(output, relPath); - if (!outputFile.exists()) { - report.addNew(); - } else if (!FileUtils.contentEquals(file, outputFile)) { - getLog().info(format("%s has changed", relPath)); - if (!outputFile.delete()) { - throw new MojoFailureException(format("can not delete %s", outputFile)); - } - report.addChanged(); - } else { - report.addUnchanged(); - } - if (!outputFile.exists()) { - File parentDir = outputFile.getParentFile(); - if (parentDir.exists() && !parentDir.isDirectory()) { - throw new MojoFailureException( - format("can not move %s to %s as %s is not a dir", file, outputFile, parentDir)); - } - if (!parentDir.exists() && !parentDir.mkdirs()) { - throw new MojoFailureException( - format("can not move %s to %s as dir %s can not be created", file, outputFile, parentDir)); - } - FileUtils.moveFile(file, outputFile); - } else { - if (!file.delete()) { - throw new MojoFailureException(format("can not delete %s", file)); - } - } - } - } - return report; - } -} diff --git a/docker/images/pinot-superset/requirements-db.txt b/docker/images/pinot-superset/requirements-db.txt index caf7d4a3ff7..322ab46c63a 100644 --- a/docker/images/pinot-superset/requirements-db.txt +++ b/docker/images/pinot-superset/requirements-db.txt @@ -17,4 +17,4 @@ # under the License. # pinotdb>=0.4.5 -redis==4.5.4 +redis>=4.6.0,<5.0 diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml index ef2ab9f1530..ce5e5df7b9b 100644 --- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml +++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml @@ -1,76 +1,90 @@ rules: -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_authorization_$2" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_authorization_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_documentsScanned_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_documentsScanned_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_entriesScannedInFilter_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_entriesScannedInFilter_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_entriesScannedPostFilter_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_entriesScannedPostFilter_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_freshnessLagMs_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_freshnessLagMs_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_queries_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_queries_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_queryExecution_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_queryExecution_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_queryRouting_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_queryRouting_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_reduce_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_reduce_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_requestCompilation_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_requestCompilation_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_scatterGather_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_scatterGather_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_totalServerResponseSize_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_totalServerResponseSize_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_groupBySize_$3" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_groupBySize_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_noServingHostForSegment_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_noServingHostForSegment_$5" cache: true labels: - table: "$1" - tableType: "$2" + database: "$2" + table: "$1$3" + tableType: "$4" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_broker_healthcheck_$1_$2" cache: true @@ -98,57 +112,67 @@ rules: - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_broker_routingTableUpdateTime_$1" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_brokerResponsesWithPartialServersResponded_$2" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_brokerResponsesWithPartialServersResponded_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_brokerResponsesWithTimeouts_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_brokerResponsesWithTimeouts_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_noServerFoundExceptions_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_noServerFoundExceptions_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_brokerResponsesWithProcessingExceptions_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_brokerResponsesWithProcessingExceptions_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_brokerResponsesWithNumGroupsLimitReached_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_brokerResponsesWithNumGroupsLimitReached_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_queryQuotaExceeded_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_queryQuotaExceeded_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_queryTotalTimeMs_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_queryTotalTimeMs_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_serverMissingForRouting_$3" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_serverMissingForRouting_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_deserialization_$2" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_deserialization_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_requestConnectionWait_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_requestConnectionWait_$4" cache: true labels: - table: "$1" + database: "$2" + table: "$1$3" - pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" name: "pinot_$1_version" cache: true @@ -157,43 +181,48 @@ rules: ## Metrics that fit the catch-all patterns above should not be added to this file. ## In case a metric does not fit the catch-all patterns, add them before this comment - # This is a catch-all pattern for pinot table metrics with offline/realtime suffix without kafka topic - # Patterns after this line may be skipped. -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$4_$5" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$2_$8" cache: true labels: - table: "$2" - tableType: "$3" -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$2_$6" + database: "$4" + table: "$3$5" + tableType: "$6" + partition: "$7" + # This is a catch-all pattern for pinot table metrics with offline/realtime suffix without kafka topic + # Patterns after this line may be skipped. +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$6_$7" cache: true labels: - table: "$3" - tableType: "$4" - partition: "$5" + database: "$3" + table: "$2$4" + tableType: "$5" #when there is no partition in the metric -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$2_$5" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$2_$7" cache: true labels: - table: "$3" - tableType: "$4" + database: "$4" + table: "$3$5" + tableType: "$6" #This is a catch-all pattern for pinot table metrics with offline/realtime suffix that also contain kafka topic -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$2_$7" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$2_$9" cache: true labels: - table: "$3" - tableType: "$4" - topic: "$5" - partition: "$6" + database: "$4" + table: "$3$5" + tableType: "$6" + topic: "$7" + partition: "$8" # This is a catch-all pattern for pinot table metrics. Patterns after this line may be skipped. -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$3_$4" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$5_$6" cache: true labels: - table: "$2" + database: "$3" + table: "$2$4" # This is a catch-all pattern for pinot controller metrics not related to tables. Patterns after this line may be skipped. - pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" name: "pinot_$1_$2_$3" diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml index 45ff802de30..44b07d1718e 100644 --- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml +++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml @@ -8,54 +8,62 @@ rules: - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_controller_helix_ZookeeperReconnects_$1" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_idealstateZnodeSize_$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_idealstateZnodeSize_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_idealstateZnodeByteSize_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_idealstateZnodeByteSize_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_replicationFromConfig_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_replicationFromConfig_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_numberOfReplicas_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_numberOfReplicas_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_percentOfReplicas_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_percentOfReplicas_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_percentSegmentsAvailable_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_percentSegmentsAvailable_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_segmentCount_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_segmentCount_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_segmentsInErrorState_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_segmentsInErrorState_$5" cache: true labels: - table: "$1" - tableType: "$2" + database: "$2" + table: "$1$3" + tableType: "$4" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_controller_numberSegmentUploadTimeoutExceeded_$1" cache: true @@ -77,128 +85,146 @@ rules: - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_controller_offlineTableCount_$1" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_validateion_$2_$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_validateion_$4_$5" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_cronSchedulerJobScheduled_$3" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_cronSchedulerJobScheduled_$5" cache: true labels: - table: "$1" - taskType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_cronSchedulerJobTriggered_$3" + database: "$2" + table: "$1$3" + taskType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_cronSchedulerJobTriggered_$5" cache: true labels: - table: "$1" - taskType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_cronSchedulerJobSkipped_$3" + database: "$2" + table: "$1$3" + taskType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_cronSchedulerJobSkipped_$5" cache: true labels: - table: "$1" - taskType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_cronSchedulerJobExecutionTimeMs_$3" + database: "$2" + table: "$1$3" + taskType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_cronSchedulerJobExecutionTimeMs_$5" cache: true labels: - table: "$1" - taskType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableRebalanceExecutionTimeMs_$3" + database: "$2" + table: "$1$3" + taskType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableRebalanceExecutionTimeMs_$5" cache: true labels: - table: "$1" - result: "$2" + database: "$2" + table: "$1$3" + result: "$4" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_controller_taskStatus_$3" cache: true labels: taskType: "$1" status: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_timeMsSinceLastMinionTaskMetadataUpdate_$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_timeMsSinceLastMinionTaskMetadataUpdate_$6" cache: true labels: - table: "$1" - tableType: "$2" - taskType: "$3" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_$1_$5" + database: "$2" + table: "$1$3" + tableType: "$4" + taskType: "$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_$1_$7" cache: true labels: - table: "$2" - tableType: "$3" - taskType: "$4" + database: "$3" + table: "$2$4" + tableType: "$5" + taskType: "$6" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_controller_$1_$3" cache: true labels: taskType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_timeMsSinceLastSuccessfulMinionTaskGeneration_$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_timeMsSinceLastSuccessfulMinionTaskGeneration_$6" cache: true labels: - table: "$1" - tableType: "$2" - taskType: "$3" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_lastMinionTaskGenerationEncountersError_$4" + database: "$2" + table: "$1$3" + tableType: "$4" + taskType: "$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_lastMinionTaskGenerationEncountersError_$6" cache: true labels: - table: "$1" - tableType: "$2" - taskType: "$3" + database: "$2" + table: "$1$3" + tableType: "$4" + taskType: "$5" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_controller_pinotLeadControllerResourceEnabled_$1" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_offlineTableEstimatedSize_$2" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_offlineTableEstimatedSize_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableQuota_$3" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableQuota_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_periodicTaskError_$4" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_periodicTaskError_$6" cache: true labels: - table: "$1" - tableType: "$2" - periodicTask: "$3" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableStorageQuotaUtilization_$3" + database: "$2" + table: "$1$3" + tableType: "$4" + periodicTask: "$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableStorageQuotaUtilization_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableStorageEstMissingSegmentPercent_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableStorageEstMissingSegmentPercent_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableTotalSizeOnServer_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableTotalSizeOnServer_$5" labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableSizePerReplicaOnServer_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableSizePerReplicaOnServer_$5" labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableCompressedSize_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableCompressedSize_$5" labels: - table: "$1" - tableType: "$2" + database: "$2" + table: "$1$3" + tableType: "$4" - pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" name: "pinot_$1_version" cache: true @@ -209,41 +235,46 @@ rules: ## In case a metric does not fit the catch-all patterns, add them before this comment # This is a catch-all pattern for pinot table metrics with offline/realtime suffix without kafka topic # Patterns after this line may be skipped. -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$4_$5" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$2_$8" cache: true labels: - table: "$2" - tableType: "$3" -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$2_$6" + database: "$4" + table: "$3$5" + tableType: "$6" + partition: "$7" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$6_$7" cache: true labels: - table: "$3" - tableType: "$4" - partition: "$5" + database: "$3" + table: "$2$4" + tableType: "$5" #This is a catch-all pattern for pinot table metrics with offline/realtime suffix that also contain kafka topic -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$2_$7" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$2_$9" cache: true labels: - table: "$3" - tableType: "$4" - topic: "$5" - partition: "$6" + database: "$4" + table: "$3$5" + tableType: "$6" + topic: "$7" + partition: "$8" #when there is no partition in the metric -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$2_$5" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$2_$7" cache: true labels: - table: "$3" - tableType: "$4" + database: "$4" + table: "$3$5" + tableType: "$6" # This is a catch-all pattern for pinot table metrics. Patterns after this line may be skipped. -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$3_$4" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$5_$6" cache: true labels: - table: "$2" + database: "$3" + table: "$2$4" # This is a catch-all pattern for pinot controller metrics not related to tables. Patterns after this line may be skipped. - pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" name: "pinot_$1_$2_$3" diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/minion.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/minion.yml index 584cde5963d..3541cc8f366 100644 --- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/minion.yml +++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/minion.yml @@ -4,19 +4,21 @@ rules: cache: true labels: version: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_minion_numberOfTasks_$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_minion_numberOfTasks_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_minion_$4_$5" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_minion_$6_$7" cache: true labels: - table: "$1" - tableType: "$2" - taskType: "$3" + database: "$2" + table: "$1$3" + tableType: "$4" + taskType: "$5" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_minion_$1_$2" cache: true @@ -36,18 +38,20 @@ rules: # This is a catch-all pattern for pinot table metrics with offline/realtime suffix. # Patterns after this line may be skipped. -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$4_$5" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$6_$7" cache: true labels: - table: "$2" - tableType: "$3" + database: "$3" + table: "$2$4" + tableType: "$5" # This is a catch-all pattern for pinot table metrics. Patterns after this line may be skipped. -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$3_$4" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$5_$6" cache: true labels: - table: "$2" + database: "$3" + table: "$2$4" # This is a catch-all pattern for pinot controller metrics not related to tables. Patterns after this line may be skipped. - pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" name: "pinot_$1_$2_$3" diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml index 6fc787cee93..2739fb557d2 100644 --- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml +++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml @@ -14,48 +14,62 @@ rules: - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_controller_helix_ZookeeperReconnects_$1" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_idealstateZnodeSize_$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_idealstateZnodeSize_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_idealstateZnodeByteSize_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_idealstateZnodeByteSize_$5" + cache: true + labels: + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_replicationFromConfig_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_numberOfReplicas_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_numberOfReplicas_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_percentOfReplicas_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_percentOfReplicas_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_percentSegmentsAvailable_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_percentSegmentsAvailable_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_segmentCount_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_segmentCount_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_segmentsInErrorState_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_segmentsInErrorState_$5" cache: true labels: - table: "$1" - tableType: "$2" + database: "$2" + table: "$1$3" + tableType: "$4" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_controller_numberSegmentUploadTimeoutExceeded_$1" cache: true @@ -77,205 +91,246 @@ rules: - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_controller_offlineTableCount_$1" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_validateion_$2_$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_validateion_$4_$5" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_cronSchedulerJobScheduled_$3" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_cronSchedulerJobScheduled_$5" cache: true labels: - table: "$1" - taskType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_cronSchedulerJobTriggered_$3" + database: "$2" + table: "$1$3" + taskType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_cronSchedulerJobTriggered_$5" cache: true labels: - table: "$1" - taskType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_cronSchedulerJobSkipped_$3" + database: "$2" + table: "$1$3" + taskType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_cronSchedulerJobSkipped_$5" cache: true labels: - table: "$1" - taskType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_cronSchedulerJobExecutionTimeMs_$3" + database: "$2" + table: "$1$3" + taskType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_cronSchedulerJobExecutionTimeMs_$5" cache: true labels: - table: "$1" - taskType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableRebalanceExecutionTimeMs_$3" + database: "$2" + table: "$1$3" + taskType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableRebalanceExecutionTimeMs_$5" cache: true labels: - table: "$1" - result: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + database: "$2" + table: "$1$3" + result: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_controller_taskStatus_$3" cache: true labels: taskType: "$1" status: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_timeMsSinceLastMinionTaskMetadataUpdate_$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_timeMsSinceLastMinionTaskMetadataUpdate_$6" cache: true labels: - table: "$1" - tableType: "$2" - taskType: "$3" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_$1_$5" + database: "$2" + table: "$1$3" + tableType: "$4" + taskType: "$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_$1_$7" cache: true labels: - table: "$2" - tableType: "$3" - taskType: "$4" + database: "$3" + table: "$2$4" + tableType: "$5" + taskType: "$6" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_controller_$1_$3" cache: true labels: taskType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_timeMsSinceLastSuccessfulMinionTaskGeneration_$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_timeMsSinceLastSuccessfulMinionTaskGeneration_$6" cache: true labels: - table: "$1" - tableType: "$2" - taskType: "$3" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_lastMinionTaskGenerationEncountersError_$4" + database: "$2" + table: "$1$3" + tableType: "$4" + taskType: "$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_lastMinionTaskGenerationEncountersError_$6" cache: true labels: - table: "$1" - tableType: "$2" - taskType: "$3" + database: "$2" + table: "$1$3" + tableType: "$4" + taskType: "$5" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_controller_pinotLeadControllerResourceEnabled_$1" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_offlineTableEstimatedSize_$2" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_offlineTableEstimatedSize_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_largestSegmentSizeOnServer_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_largestSegmentSizeOnServer_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableTotalSizeOnServer_$3" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableTotalSizeOnServer_$5" labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableSizePerReplicaOnServer_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableSizePerReplicaOnServer_$5" labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableCompressedSize_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableCompressedSize_$5" labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableQuota_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableQuota_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableStorageQuotaUtilization_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_periodicTaskError_$6" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_controller_tableStorageEstMissingSegmentPercent_$3" + database: "$2" + table: "$1$3" + tableType: "$4" + periodicTask: "$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableStorageQuotaUtilization_$5" cache: true labels: - table: "$1" - tableType: "$2" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_controller_tableStorageEstMissingSegmentPercent_$5" + cache: true + labels: + database: "$2" + table: "$1$3" + tableType: "$4" # Pinot Broker -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_authorization_$2" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_authorization_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_documentsScanned_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_documentsScanned_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_entriesScannedInFilter_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_entriesScannedInFilter_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_entriesScannedPostFilter_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_entriesScannedPostFilter_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_freshnessLagMs_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_freshnessLagMs_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_queries_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_queries_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_queryExecution_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_queryExecution_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_queryRouting_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_queryRouting_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_queryTotalTimeMs_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_queryTotalTimeMs_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_reduce_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_reduce_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_requestCompilation_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_requestCompilation_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_scatterGather_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_scatterGather_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_totalServerResponseSize_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_totalServerResponseSize_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_groupBySize_$3" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_groupBySize_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_noServingHostForSegment_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_noServingHostForSegment_$5" cache: true labels: - table: "$1" - tableType: "$2" + database: "$2" + table: "$1$3" + tableType: "$4" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_broker_healthcheck_$1_$2" cache: true @@ -303,111 +358,150 @@ rules: - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_broker_routingTableUpdateTime_$1" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_brokerResponsesWithPartialServersResponded_$2" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_brokerResponsesWithPartialServersResponded_$4" + cache: true + labels: + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_brokerResponsesWithProcessingExceptions_$4" + cache: true + labels: + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_brokerResponsesWithNumGroupsLimitReached_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_brokerResponsesWithProcessingExceptions_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_queryQuotaExceeded_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_brokerResponsesWithNumGroupsLimitReached_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_serverMissingForRouting_$5" + cache: true + labels: + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_deserialization_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_queryQuotaExceeded_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_requestConnectionWait_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_serverMissingForRouting_$3" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_brokerResponsesWithTimeouts_$4" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_deserialization_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_noServerFoundExceptions_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_broker_requestConnectionWait_$2" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_brokerResponsesWithProcessingExceptions_$4" cache: true labels: - table: "$1" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_broker_queryTotalTimeMs_$4" + cache: true + labels: + database: "$2" + table: "$1$3" # Pinot Server -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_documentCount_$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_documentCount_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_segmentCount_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_segmentCount_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_$3_$4" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_$5_$6" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_realtimeRowsConsumed_$5" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_$7_$8" cache: true labels: - table: "$1" - tableType: "$2" - topic: "$3" - partition: "$4" + database: "$2" + table: "$1$3" + tableType: "$4" + topic: "$5" + partition: "$6" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_helix_connected_$1" cache: true - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_helix_zookeeperReconnects_$1" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_highestKafkaOffsetConsumed_$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_highestKafkaOffsetConsumed_$7" cache: true labels: - table: "$1" - tableType: "$2" - topic: "$3" - partition: "$4" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_highestStreamOffsetConsumed_$5" + database: "$2" + table: "$1$3" + tableType: "$4" + topic: "$5" + partition: "$6" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_highestStreamOffsetConsumed_$7" cache: true labels: - table: "$1" - tableType: "$2" - topic: "$3" - partition: "$4" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_lastRealtimeSegment$1Seconds_$6" + database: "$2" + table: "$1$3" + tableType: "$4" + topic: "$5" + partition: "$6" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_lastRealtimeSegment$1Seconds_$8" cache: true labels: - table: "$2" - tableType: "$3" - topic: "$4" - partition: "$5" + database: "$3" + table: "$2$4" + tableType: "$5" + topic: "$6" + partition: "$7" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_llcControllerResponse_$1_$2" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_llcPartitionConsuming_$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_llcPartitionConsuming_$7" cache: true labels: - table: "$1" - tableType: "$2" - topic: "$3" - partition: "$4" + database: "$2" + table: "$1$3" + tableType: "$4" + topic: "$5" + partition: "$6" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_llcSimultaneousSegmentBuilds_$1" cache: true @@ -420,11 +514,12 @@ rules: - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_realtime_consumptionExceptions_$1" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_realtime_offheapMemoryUsed_$2" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_realtime_offheapMemoryUsed_$4" cache: true labels: - table: "$1" + database: "$2" + table: "$1$3" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_realtime_offsetCommits_$1" cache: true @@ -445,46 +540,79 @@ rules: - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_nettyConnection_$1_$2" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_realtimeSegmentNumPartitions_$2" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_realtimeSegmentNumPartitions_$4" + cache: true + labels: + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_resizeTimeMs_$5" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_resizeTimeMs_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_numResizes_$5" + cache: true + labels: + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_upsertPrimaryKeysCount_$6" + cache: true + labels: + database: "$2" + table: "$1$3" + tableType: "$4" + partition: "$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_realtimeIngestionDelayMs_$6" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_upsertPrimaryKeysCount_$4" + database: "$2" + table: "$1$3" + tableType: "$4" + partition: "$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_upsertValidDocSnapshotCount_$6" cache: true labels: - table: "$1" - tableType: "$2" - partition: "$3" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_realtimeIngestionDelayMs_$4" + database: "$2" + table: "$1$3" + tableType: "$4" + partition: "$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_upsertPrimaryKeysInSnapshotCount_$6" cache: true labels: - table: "$1" - tableType: "$2" - partition: "$3" + database: "$2" + table: "$1$3" + tableType: "$4" + partition: "$5" + #grpc related metrics +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_grpc$1_$2" + cache: true # Pinot Minions -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_minion_numberOfTasks_$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_minion_numberOfTasks_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_minion_$4_$5" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_minion_$6_$7" cache: true labels: - table: "$1" - tableType: "$2" - taskType: "$3" + database: "$2" + table: "$1$3" + tableType: "$4" + taskType: "$5" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_minion_$1_$2" cache: true @@ -503,45 +631,50 @@ rules: ## Metrics that fit the catch-all patterns above should not be added to this file. ## In case a metric does not fit the catch-all patterns, add them before this comment - # This is a catch-all pattern for pinot table metrics with offline/realtime suffix without topic or partition info + # This is a catch-all pattern for pinot table metrics with offline/realtime suffix without topic but containing partition # Patterns after this line may be skipped. -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$4_$5" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$2_$8" cache: true labels: - table: "$2" - tableType: "$3" - # This is a catch-all pattern for pinot table metrics with offline/realtime suffix without topic but containing partition + database: "$4" + table: "$3$5" + tableType: "$6" + partition: "$7" + # This is a catch-all pattern for pinot table metrics with offline/realtime suffix without topic or partition info # Patterns after this line may be skipped. -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$2_$6" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$6_$7" cache: true labels: - table: "$3" - tableType: "$4" - partition: "$5" + database: "$3" + table: "$2$4" + tableType: "$5" # This is a catch-all pattern for pinot table metrics with offline/realtime suffix with topic and partition # Patterns after this line may be skipped. -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$2_$7" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$2_$9" cache: true labels: - table: "$3" - tableType: "$4" - topic: "$5" - partition: "$6" -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$2_$5" + database: "$4" + table: "$3$5" + tableType: "$6" + topic: "$7" + partition: "$8" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$2_$7" cache: true labels: - table: "$3" - tableType: "$4" + database: "$4" + table: "$2$5" + tableType: "$6" # This is a catch-all pattern for pinot table metrics. Patterns after this line may be skipped. -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$3_$4" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$5_$6" cache: true labels: - table: "$2" + database: "$3" + table: "$2$4" # This is a catch-all pattern for pinot controller metrics not related to tables. Patterns after this line may be skipped. - pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" name: "pinot_$1_$2_$3" diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml index eb71a32a679..f59c707d68e 100644 --- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml +++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml @@ -1,70 +1,78 @@ rules: -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_documentCount_$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_documentCount_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_segmentCount_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_segmentCount_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_$3_$4" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_$5_$6" cache: true labels: - table: "$1" - tableType: "$2" + database: "$2" + table: "$1$3" + tableType: "$4" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_helix_connected_$1" cache: true - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_helix_zookeeperReconnects_$1" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_highestKafkaOffsetConsumed_$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_highestKafkaOffsetConsumed_$7" cache: true labels: - table: "$1" - tableType: "$2" - topic: "$3" - partition: "$4" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_highestStreamOffsetConsumed_$5" + database: "$2" + table: "$1$3" + tableType: "$4" + topic: "$5" + partition: "$6" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_highestStreamOffsetConsumed_$7" cache: true labels: - table: "$1" - tableType: "$2" - topic: "$3" - partition: "$4" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_lastRealtimeSegment$1Seconds_$6" + database: "$2" + table: "$1$3" + tableType: "$4" + topic: "$5" + partition: "$6" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_lastRealtimeSegment$1Seconds_$8" cache: true labels: - table: "$2" - tableType: "$3" - topic: "$4" - partition: "$5" + database: "$3" + table: "$2$4" + tableType: "$5" + topic: "$6" + partition: "$7" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_llcControllerResponse_$1_$2" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_llcPartitionConsuming_$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_llcPartitionConsuming_$7" cache: true labels: - table: "$1" - tableType: "$2" - topic: "$3" - partition: "$4" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_realtimeIngestionDelayMs_$4" + database: "$2" + table: "$1$3" + tableType: "$4" + topic: "$5" + partition: "$6" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_realtimeIngestionDelayMs_$6" cache: true labels: - table: "$1" - tableType: "$2" - partition: "$3" + database: "$2" + table: "$1$3" + tableType: "$4" + partition: "$5" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_llcSimultaneousSegmentBuilds_$1" cache: true @@ -77,19 +85,21 @@ rules: - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_realtime_consumptionExceptions_$1" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_$5_$6" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_$7_$8" cache: true labels: - table: "$1" - tableType: "$2" - topic: "$3" - partition: "$4" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_realtime_offheapMemoryUsed_$2" + database: "$2" + table: "$1$3" + tableType: "$4" + topic: "$5" + partition: "$6" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_realtime_offheapMemoryUsed_$4" cache: true labels: - table: "$1" + database: "$2" + table: "$1$3" - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_realtime_offsetCommits_$1" cache: true @@ -107,49 +117,55 @@ rules: - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_nettyConnection_$1_$2" cache: true -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_realtimeSegmentNumPartitions_$2" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_realtimeSegmentNumPartitions_$4" cache: true labels: - table: "$1" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_numResizes_$3" + database: "$2" + table: "$1$3" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_numResizes_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_resizeTimeMs_$3" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_resizeTimeMs_$5" cache: true labels: - table: "$1" - tableType: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_upsertPrimaryKeysCount_$4" + database: "$2" + table: "$1$3" + tableType: "$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_upsertPrimaryKeysCount_$6" cache: true labels: - table: "$1" - tableType: "$2" - partition: "$3" + database: "$2" + table: "$1$3" + tableType: "$4" + partition: "$5" - pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" name: "pinot_$1_version" cache: true labels: version: "$2" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_upsertValidDocSnapshotCount_$4" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_upsertValidDocSnapshotCount_$6" cache: true labels: - table: "$1" - tableType: "$2" - partition: "$3" -- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" - name: "pinot_server_upsertPrimaryKeysInSnapshotCount_$4" + database: "$2" + table: "$1$3" + tableType: "$4" + partition: "$5" +- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" + name: "pinot_server_upsertPrimaryKeysInSnapshotCount_$6" cache: true labels: - table: "$1" - tableType: "$2" - partition: "$3" + database: "$2" + table: "$1$3" + tableType: "$4" + partition: "$5" #grpc related metrics - pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)" name: "pinot_server_grpc$1_$2" @@ -157,44 +173,49 @@ rules: ## Metrics that fit the catch-all patterns above should not be added to this file. ## In case a metric does not fit the catch-all patterns, add them before this comment - # This is a catch-all pattern for pinot table metrics with offline/realtime suffix without the topic - # Patterns after this line may be skipped. -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$4_$5" + # when there is partition but no topic in the metric +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$2_$8" cache: true labels: - table: "$2" - tableType: "$3" -#when there is partition but no topic in the metric -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$2_$6" + database: "$4" + table: "$3$5" + tableType: "$6" + partition: "$7" + # This is a catch-all pattern for pinot table metrics with offline/realtime suffix without the topic + # Patterns after this line may be skipped. +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$6_$7" cache: true labels: - table: "$3" - tableType: "$4" - partition: "$5" + database: "$3" + table: "$2$4" + tableType: "$5" #when there is partition and topic in the metric -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$2_$7" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$2_$9" cache: true labels: - table: "$3" - tableType: "$4" - topic: "$5" - partition: "$6" + database: "$4" + table: "$3$5" + tableType: "$6" + topic: "$7" + partition: "$8" #when there is no partition in the metric -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$2_$5" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$2_$7" cache: true labels: - table: "$3" - tableType: "$4" + database: "$4" + table: "$3$5" + tableType: "$6" # This is a catch-all pattern for pinot table metrics. Patterns after this line may be skipped. -- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" - name: "pinot_$1_$3_$4" +- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" + name: "pinot_$1_$5_$6" cache: true labels: - table: "$2" + database: "$3" + table: "$2$4" # This is a catch-all pattern for pinot controller metrics not related to tables. Patterns after this line may be skipped. - pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<>(\\w+)" name: "pinot_$1_$2_$3" diff --git a/pinot-broker/pom.xml b/pinot-broker/pom.xml index ccc20628d42..c06903f98a3 100644 --- a/pinot-broker/pom.xml +++ b/pinot-broker/pom.xml @@ -48,45 +48,6 @@ pinot-query-runtime - - - org.glassfish.jersey.containers - jersey-container-grizzly2-http - - - org.glassfish.jersey.inject - jersey-hk2 - - - org.glassfish.jersey.media - jersey-media-json-jackson - - - org.glassfish.jersey.core - jersey-common - - - io.swagger - swagger-jaxrs - - - javax.ws.rs - jsr311-api - - - - - io.swagger - swagger-jersey2-jaxrs - - - com.jcabi - jcabi-log - - - org.glassfish.hk2 - hk2-locator - com.fasterxml.jackson.core jackson-databind diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerDebug.java b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerDebug.java index f9799d7e753..8f079ca351b 100644 --- a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerDebug.java +++ b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerDebug.java @@ -48,6 +48,7 @@ import org.apache.pinot.broker.broker.AccessControlFactory; import org.apache.pinot.broker.routing.BrokerRoutingManager; import org.apache.pinot.common.request.BrokerRequest; +import org.apache.pinot.common.utils.DatabaseUtils; import org.apache.pinot.core.auth.Actions; import org.apache.pinot.core.auth.Authorize; import org.apache.pinot.core.auth.ManualAuthorization; @@ -60,12 +61,18 @@ import org.apache.pinot.spi.utils.builder.TableNameBuilder; import org.apache.pinot.sql.parsers.CalciteSqlCompiler; +import static org.apache.pinot.spi.utils.CommonConstants.DATABASE; import static org.apache.pinot.spi.utils.CommonConstants.SWAGGER_AUTHORIZATION_KEY; -@Api(tags = "Debug", authorizations = {@Authorization(value = SWAGGER_AUTHORIZATION_KEY)}) -@SwaggerDefinition(securityDefinition = @SecurityDefinition(apiKeyAuthDefinitions = @ApiKeyAuthDefinition(name = - HttpHeaders.AUTHORIZATION, in = ApiKeyAuthDefinition.ApiKeyLocation.HEADER, key = SWAGGER_AUTHORIZATION_KEY))) +@Api(tags = "Debug", authorizations = {@Authorization(value = SWAGGER_AUTHORIZATION_KEY), + @Authorization(value = DATABASE)}) +@SwaggerDefinition(securityDefinition = @SecurityDefinition(apiKeyAuthDefinitions = { + @ApiKeyAuthDefinition(name = HttpHeaders.AUTHORIZATION, in = ApiKeyAuthDefinition.ApiKeyLocation.HEADER, + key = SWAGGER_AUTHORIZATION_KEY), + @ApiKeyAuthDefinition(name = DATABASE, in = ApiKeyAuthDefinition.ApiKeyLocation.HEADER, key = DATABASE, + description = "Database context passed through http header. If no context is provided 'default' database " + + "context will be considered.")})) @Path("/") // TODO: Add APIs to return the RoutingTable (with unavailable segments) public class PinotBrokerDebug { @@ -93,7 +100,9 @@ public class PinotBrokerDebug { @ApiResponse(code = 500, message = "Internal server error") }) public TimeBoundaryInfo getTimeBoundary( - @ApiParam(value = "Name of the table") @PathParam("tableName") String tableName) { + @ApiParam(value = "Name of the table") @PathParam("tableName") String tableName, + @Context HttpHeaders headers) { + tableName = DatabaseUtils.translateTableName(tableName, headers); String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(TableNameBuilder.extractRawTableName(tableName)); TimeBoundaryInfo timeBoundaryInfo = _routingManager.getTimeBoundaryInfo(offlineTableName); @@ -115,7 +124,9 @@ public TimeBoundaryInfo getTimeBoundary( @ApiResponse(code = 500, message = "Internal server error") }) public Map>> getRoutingTable( - @ApiParam(value = "Name of the table") @PathParam("tableName") String tableName) { + @ApiParam(value = "Name of the table") @PathParam("tableName") String tableName, + @Context HttpHeaders headers) { + tableName = DatabaseUtils.translateTableName(tableName, headers); Map>> result = new TreeMap<>(); getRoutingTable(tableName, (tableNameWithType, routingTable) -> result.put(tableNameWithType, removeOptionalSegments(routingTable.getServerInstanceToSegmentsMap()))); @@ -137,7 +148,9 @@ public Map>> getRoutingTable( @ApiResponse(code = 500, message = "Internal server error") }) public Map, List>>> getRoutingTableWithOptionalSegments( - @ApiParam(value = "Name of the table") @PathParam("tableName") String tableName) { + @ApiParam(value = "Name of the table") @PathParam("tableName") String tableName, + @Context HttpHeaders headers) { + tableName = DatabaseUtils.translateTableName(tableName, headers); Map, List>>> result = new TreeMap<>(); getRoutingTable(tableName, (tableNameWithType, routingTable) -> result.put(tableNameWithType, routingTable.getServerInstanceToSegmentsMap())); diff --git a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/BrokerCache.java b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/BrokerCache.java index 759dd32084f..6742174582d 100644 --- a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/BrokerCache.java +++ b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/BrokerCache.java @@ -28,6 +28,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Properties; import java.util.Random; import java.util.Set; @@ -190,7 +191,10 @@ protected void updateBrokerData() public String getBroker(String... tableNames) { List brokers = null; - if (tableNames != null) { + // If tableNames is not-null, filter out nulls + tableNames = + tableNames == null ? tableNames : Arrays.stream(tableNames).filter(Objects::nonNull).toArray(String[]::new); + if (!(tableNames == null || tableNames.length == 0)) { // returning list of common brokers hosting all the tables. brokers = BrokerSelectorUtils.getTablesCommonBrokers(Arrays.asList(tableNames), _brokerData.getTableToBrokerMap()); diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index 65f9778177a..a1002ca458a 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -42,6 +42,7 @@ true + org.apache.maven.plugins @@ -55,41 +56,40 @@ + org.xolstice.maven.plugins protobuf-maven-plugin + + + - - org.apache.maven.plugins - maven-dependency-plugin + maven-resources-plugin - unpack-parser-template - initialize + copy-fmpp-resources + generate-sources - unpack + copy-resources - - - org.apache.calcite - calcite-core - jar - true - ${project.build.directory}/ - **/Parser.jj,**/default_config.fmpp - - + ${project.build.directory}/codegen + + + src/main/codegen + false + + + - org.apache.pinot - pinot-fmpp-maven-plugin - ${project.version} + com.googlecode.fmpp-maven-plugin + fmpp-maven-plugin generate-fmpp-sources @@ -98,32 +98,14 @@ generate - ${project.basedir}/src/main/codegen/config.fmpp - ${project.build.directory}/generated-sources/fmpp - ${project.build.directory}/codegen/templates - tdd(${project.basedir}/src/main/codegen/config.fmpp), default:tdd(${project.build.directory}/codegen/default_config.fmpp) - - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-generated-sources - process-sources - - add-source - - - - ${project.build.directory}/generated-sources/javacc - + ${project.build.directory}/codegen/config.fmpp + ${project.build.directory}/generated-sources + ${project.build.directory}/codegen/templates + org.codehaus.mojo javacc-maven-plugin @@ -135,18 +117,19 @@ javacc - ${project.build.directory}/generated-sources/fmpp + ${project.build.directory}/generated-sources/javacc - **/Parser.jj + Parser.jj 2 false - ${project.build.directory}/generated-sources/javacc + ${project.build.directory}/generated-sources + @@ -193,12 +176,6 @@ org.apache.httpcomponents httpclient - - - commons-logging - commons-logging - - org.apache.httpcomponents @@ -212,6 +189,34 @@ org.apache.calcite calcite-babel + + org.glassfish.jersey.core + jersey-server + + + org.glassfish.jersey.containers + jersey-container-grizzly2-http + + + org.glassfish.jersey.media + jersey-media-multipart + + + org.glassfish.jersey.media + jersey-media-json-jackson + + + org.glassfish.jersey.inject + jersey-hk2 + + + org.glassfish.hk2 + hk2-metadata-generator + + + io.swagger + swagger-jersey2-jaxrs + org.testng testng @@ -307,15 +312,6 @@ org.apache.zookeeper zookeeper - - javax.servlet - javax.servlet-api - compile - - - org.glassfish.jersey.core - jersey-server - org.reflections reflections diff --git a/pinot-common/src/main/codegen/config.fmpp b/pinot-common/src/main/codegen/config.fmpp index 178029a3b8a..95c5a3d33fb 100644 --- a/pinot-common/src/main/codegen/config.fmpp +++ b/pinot-common/src/main/codegen/config.fmpp @@ -17,7 +17,11 @@ # under the License. # +# Copied from Calcite 1.36.0 babel and modified for Pinot syntax. Update this file when upgrading Calcite version. + data: { + default: tdd("../default_config.fmpp") + # Data declarations for this parser. # # Default declarations are in default_config.fmpp; if you do not include a @@ -28,24 +32,28 @@ data: { package: "org.apache.pinot.sql.parsers.parser", class: "SqlParserImpl", - # List of import statements. + # List of additional classes and packages to import. + # Example: "org.apache.calcite.sql.*", "java.util.List". imports: [ - "com.google.common.collect.*" "org.apache.pinot.sql.parsers.parser.*" - "java.util.*" ] - # List of new keywords to add + # List of new keywords. Example: "DATABASES", "TABLES". If the keyword is + # not a reserved keyword, add it to the 'nonReservedKeywords' section. keywords: [ "FILE" "ARCHIVE" ] - # List of non-reserved keywords to add + # List of non-reserved keywords to add; + # items in this list become non-reserved nonReservedKeywordsToAdd: [ - # customized for Pinot "FILE" "ARCHIVE" + # Pinot allows using DEFAULT as the catalog name + "DEFAULT_" + # Pinot allows using DATETIME as column name + "DATETIME" # The following keywords are reserved in core Calcite, # are reserved in some version of SQL, @@ -132,6 +140,7 @@ data: { "CONSTRAINTS" "CONSTRUCTOR" "CONTAINS" + "CONTAINS_SUBSTR" "CONTINUE" "CONVERT" "CORR" @@ -159,12 +168,13 @@ data: { "CYCLE" "DATA" # "DATE" + "DATETIME_DIFF" "DAY" "DEALLOCATE" "DEC" "DECIMAL" "DECLARE" - "DEFAULT_" +# "DEFAULT" "DEFERRABLE" "DEFERRED" # "DEFINE" @@ -239,7 +249,6 @@ data: { "HOLD" "HOUR" "IDENTITY" -# "IF" # not a keyword in Calcite "ILIKE" "IMMEDIATE" "IMMEDIATELY" @@ -466,7 +475,11 @@ data: { "TEMPORARY" # "THEN" # "TIME" + "TIME_DIFF" + "TIME_TRUNC" # "TIMESTAMP" + "TIMESTAMP_DIFF" + "TIMESTAMP_TRUNC" "TIMEZONE_HOUR" "TIMEZONE_MINUTE" "TINYINT" @@ -523,21 +536,99 @@ data: { "ZONE" ] - # List of extended statement syntax to add + # List of non-reserved keywords to remove; + # items in this list become reserved. + nonReservedKeywordsToRemove: [ + ] + + # List of additional join types. Each is a method with no arguments. + # Example: "LeftSemiJoin". + joinTypes: [ + ] + + # List of methods for parsing custom SQL statements. + # Return type of method implementation should be 'SqlNode'. + # Example: "SqlShowDatabases()", "SqlShowTables()". statementParserMethods: [ "SqlInsertFromFile()" "SqlPhysicalExplain()" ] - # List of custom function syntax to add + # List of methods for parsing custom literals. + # Return type of method implementation should be "SqlNode". + # Example: ParseJsonLiteral(). + literalParserMethods: [ + ] + + # List of methods for parsing custom data types. + # Return type of method implementation should be "SqlTypeNameSpec". + # Example: SqlParseTimeStampZ(). + dataTypeParserMethods: [ + ] + + # List of methods for parsing builtin function calls. + # Return type of method implementation should be "SqlNode". + # Example: "DateTimeConstructorCall()". + builtinFunctionCallMethods: [ + ] + + # List of methods for parsing extensions to "ALTER " calls. + # Each must accept arguments "(SqlParserPos pos, String scope)". + # Example: "SqlAlterTable". + alterStatementParserMethods: [ + ] + + # List of methods for parsing extensions to "CREATE [OR REPLACE]" calls. + # Each must accept arguments "(SqlParserPos pos, boolean replace)". + # Example: "SqlCreateForeignSchema". + createStatementParserMethods: [ + ] + + # List of methods for parsing extensions to "DROP" calls. + # Each must accept arguments "(SqlParserPos pos)". + # Example: "SqlDropSchema". + dropStatementParserMethods: [ + ] + + # List of methods for parsing extensions to "TRUNCATE" calls. + # Each must accept arguments "(SqlParserPos pos)". + # Example: "SqlTruncate". + truncateStatementParserMethods: [ + ] + + # Binary operators tokens. + # Example: "< INFIX_CAST: \"::\" >". + binaryOperatorsTokens: [ + ] + + # Binary operators initialization. + # Example: "InfixCast". extraBinaryExpressions: [ "SqlAtTimeZone" ] # List of files in @includes directory that have parser method + # implementations for parsing custom SQL statements, literals or types + # given as part of "statementParserMethods", "literalParserMethods" or + # "dataTypeParserMethods". + # Example: "parserImpls.ftl". implementationFiles: [ "parserImpls.ftl" - ], + ] + + # Custom identifier token. + # + # PostgreSQL allows letters with diacritical marks and non-Latin letters + # in the beginning of identifier and additionally dollar sign in the rest of identifier. + # Letters with diacritical marks and non-Latin letters + # are represented by character codes 128 to 255 (or in octal \200 to \377). + # See https://learn.microsoft.com/en-gb/office/vba/language/reference/user-interface-help/character-set-128255 + # See https://github.com/postgres/postgres/blob/master/src/backend/parser/scan.l + # + # MySQL allows digit in the beginning of identifier + customIdentifierToken: "< IDENTIFIER: (||[\"\\200\"-\"\\377\"]) (|||[\"\\200\"-\"\\377\"])* >" + + includeParsingStringLiteralAsArrayLiteral: true } } diff --git a/pinot-common/src/main/codegen/default_config.fmpp b/pinot-common/src/main/codegen/default_config.fmpp new file mode 100644 index 00000000000..78191c0c11c --- /dev/null +++ b/pinot-common/src/main/codegen/default_config.fmpp @@ -0,0 +1,465 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Copied from Calcite 1.36.0 and modified for Pinot syntax. Update this file when upgrading Calcite version. + +# Default data declarations for parsers. +# Each of these may be overridden in a parser's config.fmpp file. +# In addition, each parser must define "package" and "class". +parser: { + # List of additional classes and packages to import. + # Example: "org.apache.calcite.sql.*", "java.util.List". + imports: [ + ] + + # List of new keywords. Example: "DATABASES", "TABLES". If the keyword is + # not a reserved keyword, add it to the 'nonReservedKeywords' section. + keywords: [ + ] + + # List of keywords from "keywords" section that are not reserved. + nonReservedKeywords: [ + "A" + "ABSENT" + "ABSOLUTE" + "ACTION" + "ADA" + "ADD" + "ADMIN" + "AFTER" + "ALWAYS" + "APPLY" + "ARRAY_AGG" + "ARRAY_CONCAT_AGG" + "ASC" + "ASSERTION" + "ASSIGNMENT" + "ATTRIBUTE" + "ATTRIBUTES" + "BEFORE" + "BERNOULLI" + "BREADTH" + "C" + "CASCADE" + "CATALOG" + "CATALOG_NAME" + "CENTURY" + "CHAIN" + "CHARACTERISTICS" + "CHARACTERS" + "CHARACTER_SET_CATALOG" + "CHARACTER_SET_NAME" + "CHARACTER_SET_SCHEMA" + "CLASS_ORIGIN" + "COBOL" + "COLLATION" + "COLLATION_CATALOG" + "COLLATION_NAME" + "COLLATION_SCHEMA" + "COLUMN_NAME" + "COMMAND_FUNCTION" + "COMMAND_FUNCTION_CODE" + "COMMITTED" + "CONDITIONAL" + "CONDITION_NUMBER" + "CONNECTION" + "CONNECTION_NAME" + "CONSTRAINT_CATALOG" + "CONSTRAINT_NAME" + "CONSTRAINTS" + "CONSTRAINT_SCHEMA" + "CONSTRUCTOR" + "CONTAINS_SUBSTR" + "CONTINUE" + "CURSOR_NAME" + "DATA" + "DATABASE" + "DATE_DIFF" + "DATE_TRUNC" + "DATETIME_DIFF" + "DATETIME_INTERVAL_CODE" + "DATETIME_INTERVAL_PRECISION" + "DATETIME_TRUNC" + "DAYOFWEEK" + "DAYOFYEAR" + "DAYS" + "DECADE" + "DEFAULTS" + "DEFERRABLE" + "DEFERRED" + "DEFINED" + "DEFINER" + "DEGREE" + "DEPTH" + "DERIVED" + "DESC" + "DESCRIPTION" + "DESCRIPTOR" + "DIAGNOSTICS" + "DISPATCH" + "DOMAIN" + "DOW" + "DOY" + "DOT_FORMAT" + "DYNAMIC_FUNCTION" + "DYNAMIC_FUNCTION_CODE" + "ENCODING" + "EPOCH" + "ERROR" + "EXCEPTION" + "EXCLUDE" + "EXCLUDING" + "FINAL" + "FIRST" + "FOLLOWING" + "FORMAT" + "FORTRAN" + "FOUND" + "FRAC_SECOND" + "G" + "GENERAL" + "GENERATED" + "GEOMETRY" + "GO" + "GOTO" + "GRANTED" + "GROUP_CONCAT" + "HIERARCHY" + "HOP" + "HOURS" + "IGNORE" + "ILIKE" + "IMMEDIATE" + "IMMEDIATELY" + "IMPLEMENTATION" + "INCLUDE" + "INCLUDING" + "INCREMENT" + "INITIALLY" + "INPUT" + "INSTANCE" + "INSTANTIABLE" + "INVOKER" + "ISODOW" + "ISOLATION" + "ISOYEAR" + "JAVA" + "JSON" + "K" + "KEY" + "KEY_MEMBER" + "KEY_TYPE" + "LABEL" + "LAST" + "LENGTH" + "LEVEL" + "LIBRARY" + "LOCATOR" + "M" + "MAP" + "MATCHED" + "MAXVALUE" + "MESSAGE_LENGTH" + "MESSAGE_OCTET_LENGTH" + "MESSAGE_TEXT" + "MICROSECOND" + "MILLENNIUM" + "MILLISECOND" + "MINUTES" + "MINVALUE" + "MONTHS" + "MORE_" + "MUMPS" + "NAME" + "NAMES" + "NANOSECOND" + "NESTING" + "NORMALIZED" + "NULLABLE" + "NULLS" + "NUMBER" + "OBJECT" + "OCTETS" + "OPTION" + "OPTIONS" + "ORDERING" + "ORDINALITY" + "OTHERS" + "OUTPUT" + "OVERRIDING" + "PAD" + "PARAMETER_MODE" + "PARAMETER_NAME" + "PARAMETER_ORDINAL_POSITION" + "PARAMETER_SPECIFIC_CATALOG" + "PARAMETER_SPECIFIC_NAME" + "PARAMETER_SPECIFIC_SCHEMA" + "PARTIAL" + "PASCAL" + "PASSING" + "PASSTHROUGH" + "PAST" + "PATH" + "PIVOT" + "PLACING" + "PLAN" + "PLI" + "PRECEDING" + "PRESERVE" + "PRIOR" + "PRIVILEGES" + "PUBLIC" + "QUARTER" + "QUARTERS" + "READ" + "RELATIVE" + "REPEATABLE" + "REPLACE" + "RESPECT" + "RESTART" + "RESTRICT" + "RETURNED_CARDINALITY" + "RETURNED_LENGTH" + "RETURNED_OCTET_LENGTH" + "RETURNED_SQLSTATE" + "RETURNING" + "RLIKE" + "ROLE" + "ROUTINE" + "ROUTINE_CATALOG" + "ROUTINE_NAME" + "ROUTINE_SCHEMA" + "ROW_COUNT" + "SCALAR" + "SCALE" + "SCHEMA" + "SCHEMA_NAME" + "SCOPE_CATALOGS" + "SCOPE_NAME" + "SCOPE_SCHEMA" + "SECONDS" + "SECTION" + "SECURITY" + "SELF" + "SEPARATOR" + "SEQUENCE" + "SERIALIZABLE" + "SERVER" + "SERVER_NAME" + "SESSION" + "SETS" + "SIMPLE" + "SIZE" + "SOURCE" + "SPACE" + "SPECIFIC_NAME" + "SQL_BIGINT" + "SQL_BINARY" + "SQL_BIT" + "SQL_BLOB" + "SQL_BOOLEAN" + "SQL_CHAR" + "SQL_CLOB" + "SQL_DATE" + "SQL_DECIMAL" + "SQL_DOUBLE" + "SQL_FLOAT" + "SQL_INTEGER" + "SQL_INTERVAL_DAY" + "SQL_INTERVAL_DAY_TO_HOUR" + "SQL_INTERVAL_DAY_TO_MINUTE" + "SQL_INTERVAL_DAY_TO_SECOND" + "SQL_INTERVAL_HOUR" + "SQL_INTERVAL_HOUR_TO_MINUTE" + "SQL_INTERVAL_HOUR_TO_SECOND" + "SQL_INTERVAL_MINUTE" + "SQL_INTERVAL_MINUTE_TO_SECOND" + "SQL_INTERVAL_MONTH" + "SQL_INTERVAL_SECOND" + "SQL_INTERVAL_YEAR" + "SQL_INTERVAL_YEAR_TO_MONTH" + "SQL_LONGVARBINARY" + "SQL_LONGVARCHAR" + "SQL_LONGVARNCHAR" + "SQL_NCHAR" + "SQL_NCLOB" + "SQL_NUMERIC" + "SQL_NVARCHAR" + "SQL_REAL" + "SQL_SMALLINT" + "SQL_TIME" + "SQL_TIMESTAMP" + "SQL_TINYINT" + "SQL_TSI_DAY" + "SQL_TSI_FRAC_SECOND" + "SQL_TSI_HOUR" + "SQL_TSI_MICROSECOND" + "SQL_TSI_MINUTE" + "SQL_TSI_MONTH" + "SQL_TSI_QUARTER" + "SQL_TSI_SECOND" + "SQL_TSI_WEEK" + "SQL_TSI_YEAR" + "SQL_VARBINARY" + "SQL_VARCHAR" + "STATE" + "STATEMENT" + "STRING_AGG" + "STRUCTURE" + "STYLE" + "SUBCLASS_ORIGIN" + "SUBSTITUTE" + "TABLE_NAME" + "TEMPORARY" + "TIES" + "TIME_DIFF" + "TIME_TRUNC" + "TIMESTAMPADD" + "TIMESTAMPDIFF" + "TIMESTAMP_DIFF" + "TIMESTAMP_TRUNC" + "TOP_LEVEL_COUNT" + "TRANSACTION" + "TRANSACTIONS_ACTIVE" + "TRANSACTIONS_COMMITTED" + "TRANSACTIONS_ROLLED_BACK" + "TRANSFORM" + "TRANSFORMS" + "TRIGGER_CATALOG" + "TRIGGER_NAME" + "TRIGGER_SCHEMA" + "TUMBLE" + "TYPE" + "UNBOUNDED" + "UNCOMMITTED" + "UNCONDITIONAL" + "UNDER" + "UNPIVOT" + "UNNAMED" + "USAGE" + "USER_DEFINED_TYPE_CATALOG" + "USER_DEFINED_TYPE_CODE" + "USER_DEFINED_TYPE_NAME" + "USER_DEFINED_TYPE_SCHEMA" + "UTF16" + "UTF32" + "UTF8" + "VERSION" + "VIEW" + "WEEK" + "WEEKS" + "WORK" + "WRAPPER" + "WRITE" + "XML" + "YEARS" + "ZONE" + ] + + # List of non-reserved keywords to add; + # items in this list become non-reserved. + nonReservedKeywordsToAdd: [ + ] + + # List of non-reserved keywords to remove; + # items in this list become reserved. + nonReservedKeywordsToRemove: [ + ] + + # List of additional join types. Each is a method with no arguments. + # Example: "LeftSemiJoin". + joinTypes: [ + ] + + # List of methods for parsing custom SQL statements. + # Return type of method implementation should be 'SqlNode'. + # Example: "SqlShowDatabases()", "SqlShowTables()". + statementParserMethods: [ + ] + + # List of methods for parsing custom literals. + # Return type of method implementation should be "SqlNode". + # Example: ParseJsonLiteral(). + literalParserMethods: [ + ] + + # List of methods for parsing custom data types. + # Return type of method implementation should be "SqlTypeNameSpec". + # Example: SqlParseTimeStampZ(). + dataTypeParserMethods: [ + ] + + # List of methods for parsing builtin function calls. + # Return type of method implementation should be "SqlNode". + # Example: "DateTimeConstructorCall()". + builtinFunctionCallMethods: [ + ] + + # List of methods for parsing extensions to "ALTER " calls. + # Each must accept arguments "(SqlParserPos pos, String scope)". + # Example: "SqlAlterTable". + alterStatementParserMethods: [ + ] + + # List of methods for parsing extensions to "CREATE [OR REPLACE]" calls. + # Each must accept arguments "(SqlParserPos pos, boolean replace)". + # Example: "SqlCreateForeignSchema". + createStatementParserMethods: [ + ] + + # List of methods for parsing extensions to "DROP" calls. + # Each must accept arguments "(SqlParserPos pos)". + # Example: "SqlDropSchema". + dropStatementParserMethods: [ + ] + + # List of methods for parsing extensions to "TRUNCATE" calls. + # Each must accept arguments "(SqlParserPos pos)". + # Example: "SqlTruncate". + truncateStatementParserMethods: [ + ] + + # Binary operators tokens. + # Example: "< INFIX_CAST: \"::\" >". + binaryOperatorsTokens: [ + ] + + # Binary operators initialization. + # Example: "InfixCast". + extraBinaryExpressions: [ + ] + + # List of files in @includes directory that have parser method + # implementations for parsing custom SQL statements, literals or types + # given as part of "statementParserMethods", "literalParserMethods" or + # "dataTypeParserMethods". + # Example: "parserImpls.ftl". + implementationFiles: [ + ] + + # Custom identifier token. + # Example: "< IDENTIFIER: (|)+ >". + customIdentifierToken: "" + + includePosixOperators: false + includeCompoundIdentifier: true + includeBraces: true + includeAdditionalDeclarations: false + includeParsingStringLiteralAsArrayLiteral: false +} diff --git a/pinot-common/src/main/codegen/includes/parserImpls.ftl b/pinot-common/src/main/codegen/includes/parserImpls.ftl index 989894dd5db..6e1283b075b 100644 --- a/pinot-common/src/main/codegen/includes/parserImpls.ftl +++ b/pinot-common/src/main/codegen/includes/parserImpls.ftl @@ -33,7 +33,7 @@ private void DataFileDef(List list) : SqlNodeList DataFileDefList() : { SqlParserPos pos; - List list = Lists.newArrayList(); + List list = new ArrayList(); } { { pos = getPos(); } @@ -73,36 +73,6 @@ SqlInsertFromFile SqlInsertFromFile() : } } -/** - * define the rest of the sql into SqlStmtList - */ -private void SqlStatementList(SqlNodeList list) : -{ -} -{ - { - list.add(SqlStmt()); - } -} - -SqlNodeList SqlStmtsEof() : -{ - SqlParserPos pos; - SqlNodeList stmts; -} -{ - { - pos = getPos(); - stmts = new SqlNodeList(pos); - stmts.add(SqlStmt()); - } - ( LOOKAHEAD(2, SqlStmt()) SqlStatementList(stmts) )* - [ ] - { - return stmts; - } -} - void SqlAtTimeZone(List list, ExprContext exprContext, Span s) : { List list2; diff --git a/pinot-common/src/main/codegen/templates/Parser.jj b/pinot-common/src/main/codegen/templates/Parser.jj new file mode 100644 index 00000000000..1e86c8c3943 --- /dev/null +++ b/pinot-common/src/main/codegen/templates/Parser.jj @@ -0,0 +1,9131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Copied from Calcite 1.36.0 and modified for Pinot syntax. Update this file when upgrading Calcite version. +// Modified parts are marked with "PINOT CUSTOMIZATION START/END". + +<@pp.dropOutputFile /> + +<@pp.changeOutputFile name="javacc/Parser.jj" /> + +options { + STATIC = false; + IGNORE_CASE = true; + UNICODE_INPUT = true; +} + + +PARSER_BEGIN(${parser.class}) + +package ${parser.package}; + +<#list (parser.imports!default.parser.imports) as importStr> +import ${importStr}; + + +import org.apache.calcite.avatica.util.Casing; +import org.apache.calcite.avatica.util.TimeUnit; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.runtime.CalciteContextException; +import org.apache.calcite.sql.JoinConditionType; +import org.apache.calcite.sql.JoinType; +import org.apache.calcite.sql.SqlAlter; +import org.apache.calcite.sql.SqlBasicTypeNameSpec; +import org.apache.calcite.sql.SqlBinaryOperator; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlCharStringLiteral; +import org.apache.calcite.sql.SqlCollation; +import org.apache.calcite.sql.SqlCollectionTypeNameSpec; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlDelete; +import org.apache.calcite.sql.SqlDescribeSchema; +import org.apache.calcite.sql.SqlDescribeTable; +import org.apache.calcite.sql.SqlDynamicParam; +import org.apache.calcite.sql.SqlExplain; +import org.apache.calcite.sql.SqlExplainFormat; +import org.apache.calcite.sql.SqlExplainLevel; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlHint; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlInsert; +import org.apache.calcite.sql.SqlInsertKeyword; +import org.apache.calcite.sql.SqlIntervalQualifier; +import org.apache.calcite.sql.SqlJdbcDataTypeName; +import org.apache.calcite.sql.SqlJdbcFunctionCall; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlJsonConstructorNullClause; +import org.apache.calcite.sql.SqlJsonEncoding; +import org.apache.calcite.sql.SqlJsonExistsErrorBehavior; +import org.apache.calcite.sql.SqlJsonEmptyOrError; +import org.apache.calcite.sql.SqlJsonQueryEmptyOrErrorBehavior; +import org.apache.calcite.sql.SqlJsonQueryWrapperBehavior; +import org.apache.calcite.sql.SqlJsonValueEmptyOrErrorBehavior; +import org.apache.calcite.sql.SqlJsonValueReturning; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlMatchRecognize; +import org.apache.calcite.sql.SqlMerge; +import org.apache.calcite.sql.SqlMapTypeNameSpec; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlNumericLiteral; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlOrderBy; +import org.apache.calcite.sql.SqlPivot; +import org.apache.calcite.sql.SqlPostfixOperator; +import org.apache.calcite.sql.SqlPrefixOperator; +import org.apache.calcite.sql.SqlRowTypeNameSpec; +import org.apache.calcite.sql.SqlSampleSpec; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlSelectKeyword; +import org.apache.calcite.sql.SqlSetOption; +import org.apache.calcite.sql.SqlSnapshot; +import org.apache.calcite.sql.SqlTableRef; +import org.apache.calcite.sql.SqlTypeNameSpec; +import org.apache.calcite.sql.SqlUnnestOperator; +import org.apache.calcite.sql.SqlUnpivot; +import org.apache.calcite.sql.SqlUpdate; +import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; +import org.apache.calcite.sql.SqlUtil; +import org.apache.calcite.sql.SqlWindow; +import org.apache.calcite.sql.SqlWith; +import org.apache.calcite.sql.SqlWithItem; +import org.apache.calcite.sql.fun.SqlCase; +import org.apache.calcite.sql.fun.SqlInternalOperators; +import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.fun.SqlTrimFunction; +import org.apache.calcite.sql.parser.Span; +import org.apache.calcite.sql.parser.SqlAbstractParserImpl; +import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.parser.SqlParserImplFactory; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.parser.SqlParserUtil; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.validate.SqlConformance; +import org.apache.calcite.sql.validate.SqlConformanceEnum; +import org.apache.calcite.util.Glossary; +import org.apache.calcite.util.Pair; +import org.apache.calcite.util.SourceStringReader; +import org.apache.calcite.util.Util; +import org.apache.calcite.util.trace.CalciteTrace; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.slf4j.Logger; + +import java.io.Reader; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.apache.calcite.util.Static.RESOURCE; + +/** + * SQL parser, generated from Parser.jj by JavaCC. + * + *

The public wrapper for this parser is {@link SqlParser}. + */ +public class ${parser.class} extends SqlAbstractParserImpl +{ + private static final Logger LOGGER = CalciteTrace.getParserTracer(); + + // Can't use quoted literal because of a bug in how JavaCC translates + // backslash-backslash. + private static final char BACKSLASH = 0x5c; + private static final char DOUBLE_QUOTE = 0x22; + private static final String DQ = DOUBLE_QUOTE + ""; + private static final String DQDQ = DQ + DQ; + private static final SqlLiteral LITERAL_ZERO = + SqlLiteral.createExactNumeric("0", SqlParserPos.ZERO); + private static final SqlLiteral LITERAL_ONE = + SqlLiteral.createExactNumeric("1", SqlParserPos.ZERO); + private static final SqlLiteral LITERAL_MINUS_ONE = + SqlLiteral.createExactNumeric("-1", SqlParserPos.ZERO); + private static final BigDecimal ONE_HUNDRED = BigDecimal.valueOf(100L); + + private static Metadata metadata; + + private Casing unquotedCasing; + private Casing quotedCasing; + private int identifierMaxLength; + private SqlConformance conformance; + + /** + * {@link SqlParserImplFactory} implementation for creating parser. + */ + public static final SqlParserImplFactory FACTORY = new SqlParserImplFactory() { + public SqlAbstractParserImpl getParser(Reader reader) { + final ${parser.class} parser = new ${parser.class}(reader); + if (reader instanceof SourceStringReader) { + final String sql = + ((SourceStringReader) reader).getSourceString(); + parser.setOriginalSql(sql); + } + return parser; + } + }; + + public SqlParseException normalizeException(Throwable ex) { + try { + if (ex instanceof ParseException) { + ex = cleanupParseException((ParseException) ex); + } + return convertException(ex); + } catch (ParseException e) { + throw new AssertionError(e); + } + } + + public Metadata getMetadata() { + synchronized (${parser.class}.class) { + if (metadata == null) { + metadata = new MetadataImpl( + new ${parser.class}(new java.io.StringReader(""))); + } + return metadata; + } + } + + public void setTabSize(int tabSize) { + jj_input_stream.setTabSize(tabSize); + } + + public void switchTo(SqlAbstractParserImpl.LexicalState state) { + final int stateOrdinal = + Arrays.asList(${parser.class}TokenManager.lexStateNames) + .indexOf(state.name()); + token_source.SwitchTo(stateOrdinal); + } + + public void setQuotedCasing(Casing quotedCasing) { + this.quotedCasing = quotedCasing; + } + + public void setUnquotedCasing(Casing unquotedCasing) { + this.unquotedCasing = unquotedCasing; + } + + public void setIdentifierMaxLength(int identifierMaxLength) { + this.identifierMaxLength = identifierMaxLength; + } + + public void setConformance(SqlConformance conformance) { + this.conformance = conformance; + } + + public SqlNode parseSqlExpressionEof() throws Exception { + return SqlExpressionEof(); + } + + public SqlNode parseSqlStmtEof() throws Exception { + return SqlStmtEof(); + } + + public SqlNodeList parseSqlStmtList() throws Exception { + return SqlStmtList(); + } + + public SqlNode parseArray() throws SqlParseException { + switchTo(LexicalState.BQID); + try { + return ArrayLiteral(); + } catch (ParseException ex) { + throw normalizeException(ex); + } catch (TokenMgrError ex) { + throw normalizeException(ex); + } + } + + private SqlNode extend(SqlNode table, SqlNodeList extendList) { + return SqlStdOperatorTable.EXTEND.createCall( + Span.of(table, extendList).pos(), table, extendList); + } + + /** Adds a warning that a token such as "HOURS" was used, + * whereas the SQL standard only allows "HOUR". + * + *

Currently, we silently add an exception to a list of warnings. In + * future, we may have better compliance checking, for example a strict + * compliance mode that throws if any non-standard features are used. */ + private TimeUnit warn(TimeUnit timeUnit) throws ParseException { + final String token = getToken(0).image.toUpperCase(Locale.ROOT); + warnings.add( + SqlUtil.newContextException(getPos(), + RESOURCE.nonStandardFeatureUsed(token))); + return timeUnit; + } +} + +PARSER_END(${parser.class}) + + +/*************************************** + * Utility Codes for Semantic Analysis * + ***************************************/ + +/* For Debug */ +JAVACODE +void debug_message1() { + LOGGER.info("{} , {}", getToken(0).image, getToken(1).image); +} + +JAVACODE String unquotedIdentifier() { + return SqlParserUtil.toCase(getToken(0).image, unquotedCasing); +} + +/** + * Allows parser to be extended with new types of table references. The + * default implementation of this production is empty. + */ +SqlNode ExtendedTableRef() : +{ +} +{ + UnusedExtension() + { + return null; + } +} + +/** + * Allows an OVER clause following a table expression as an extension to + * standard SQL syntax. The default implementation of this production is empty. + */ +SqlNode TableOverOpt() : +{ +} +{ + { + return null; + } +} + +/* + * Parses dialect-specific keywords immediately following the SELECT keyword. + */ +void SqlSelectKeywords(List keywords) : +{} +{ + E() +} + +/* + * Parses dialect-specific keywords immediately following the INSERT keyword. + */ +void SqlInsertKeywords(List keywords) : +{} +{ + E() +} + +/* +* Parse Floor/Ceil function parameters +*/ +SqlNode FloorCeilOptions(Span s, boolean floorFlag) : +{ + SqlNode node; +} +{ + node = StandardFloorCeilOptions(s, floorFlag) { + return node; + } +} + +/* +// This file contains the heart of a parser for SQL SELECT statements. +// code can be shared between various parsers (for example, a DDL parser and a +// DML parser) but is not a standalone JavaCC file. You need to prepend a +// parser declaration (such as that in Parser.jj). +*/ + +/* Epsilon */ +JAVACODE +void E() {} + +/** @Deprecated */ +JAVACODE List startList(Object o) +{ + List list = new ArrayList(); + list.add(o); + return list; +} + +/* + * NOTE jvs 6-Feb-2004: The straightforward way to implement the SQL grammar is + * to keep query expressions (SELECT, UNION, etc) separate from row expressions + * (+, LIKE, etc). However, this is not possible with an LL(k) parser, because + * both kinds of expressions allow parenthesization, so no fixed amount of left + * context is ever good enough. A sub-query can be a leaf in a row expression, + * and can include operators like UNION, so it's not even possible to use a + * syntactic lookahead rule like "look past an indefinite number of parentheses + * until you see SELECT, VALUES, or TABLE" (since at that point we still + * don't know whether we're parsing a sub-query like ((select ...) + x) + * vs. (select ... union select ...). + * + * The somewhat messy solution is to unify the two kinds of expression, + * and to enforce syntax rules using parameterized context. This + * is the purpose of the ExprContext parameter. It is passed to + * most expression productions, which check the expressions encountered + * against the context for correctness. When a query + * element like SELECT is encountered, the production calls + * checkQueryExpression, which will throw an exception if + * a row expression was expected instead. When a row expression like + * IN is encountered, the production calls checkNonQueryExpression + * instead. It is very important to understand how this works + * when modifying the grammar. + * + * The commingling of expressions results in some bogus ambiguities which are + * resolved with LOOKAHEAD hints. The worst example is comma. SQL allows both + * (WHERE x IN (1,2)) and (WHERE x IN (select ...)). This means when we parse + * the right-hand-side of an IN, we have to allow any kind of expression inside + * the parentheses. Now consider the expression "WHERE x IN(SELECT a FROM b + * GROUP BY c,d)". When the parser gets to "c,d" it doesn't know whether the + * comma indicates the end of the GROUP BY or the end of one item in an IN + * list. Luckily, we know that select and comma-list are mutually exclusive + * within IN, so we use maximal munch for the GROUP BY comma. However, this + * usage of hints could easily mask unintended ambiguities resulting from + * future changes to the grammar, making it very brittle. + */ + +JAVACODE protected SqlParserPos getPos() +{ + return new SqlParserPos( + token.beginLine, + token.beginColumn, + token.endLine, + token.endColumn); +} + +/** Starts a span at the current position. */ +JAVACODE Span span() +{ + return Span.of(getPos()); +} + +JAVACODE void checkQueryExpression(ExprContext exprContext) +{ + switch (exprContext) { + case ACCEPT_NON_QUERY: + case ACCEPT_SUB_QUERY: + case ACCEPT_CURSOR: + throw SqlUtil.newContextException(getPos(), + RESOURCE.illegalQueryExpression()); + } +} + +JAVACODE void checkNonQueryExpression(ExprContext exprContext) +{ + switch (exprContext) { + case ACCEPT_QUERY: + throw SqlUtil.newContextException(getPos(), + RESOURCE.illegalNonQueryExpression()); + } +} + +JAVACODE SqlNode checkNotJoin(SqlNode e) +{ + if (e instanceof SqlJoin) { + throw SqlUtil.newContextException(e.getParserPosition(), + RESOURCE.illegalJoinExpression()); + } + return e; +} + +/** + * Converts a ParseException (local to this particular instantiation + * of the parser) into a SqlParseException (common to all parsers). + */ +JAVACODE SqlParseException convertException(Throwable ex) +{ + if (ex instanceof SqlParseException) { + return (SqlParseException) ex; + } + SqlParserPos pos = null; + int[][] expectedTokenSequences = null; + String[] tokenImage = null; + if (ex instanceof ParseException) { + ParseException pex = (ParseException) ex; + expectedTokenSequences = pex.expectedTokenSequences; + tokenImage = pex.tokenImage; + if (pex.currentToken != null) { + final Token token = pex.currentToken.next; + // Checks token.image.equals("1") to avoid recursive call. + // The SqlAbstractParserImpl#MetadataImpl constructor uses constant "1" to + // throw intentionally to collect the expected tokens. + if (!token.image.equals("1") + && getMetadata().isKeyword(token.image) + && SqlParserUtil.allowsIdentifier(tokenImage, expectedTokenSequences)) { + // If the next token is a keyword, reformat the error message as: + + // Incorrect syntax near the keyword '{keyword}' at line {line_number}, + // column {column_number}. + final String expecting = ex.getMessage() + .substring(ex.getMessage().indexOf("Was expecting")); + final String errorMsg = String.format("Incorrect syntax near the keyword '%s' " + + "at line %d, column %d.\n%s", + token.image, + token.beginLine, + token.beginColumn, + expecting); + // Replace the ParseException with explicit error message. + ex = new ParseException(errorMsg); + } + pos = new SqlParserPos( + token.beginLine, + token.beginColumn, + token.endLine, + token.endColumn); + } + } else if (ex instanceof TokenMgrError) { + expectedTokenSequences = null; + tokenImage = null; + // Example: + // Lexical error at line 3, column 24. Encountered "#" after "a". + final java.util.regex.Pattern pattern = java.util.regex.Pattern.compile( + "(?s)Lexical error at line ([0-9]+), column ([0-9]+).*"); + java.util.regex.Matcher matcher = pattern.matcher(ex.getMessage()); + if (matcher.matches()) { + int line = Integer.parseInt(matcher.group(1)); + int column = Integer.parseInt(matcher.group(2)); + pos = new SqlParserPos(line, column, line, column); + } + } else if (ex instanceof CalciteContextException) { + // CalciteContextException is the standard wrapper for exceptions + // produced by the validator, but in the parser, the standard is + // SqlParseException; so, strip it away. In case you were wondering, + // the CalciteContextException appears because the parser + // occasionally calls into validator-style code such as + // SqlSpecialOperator.reduceExpr. + CalciteContextException ece = + (CalciteContextException) ex; + pos = new SqlParserPos( + ece.getPosLine(), + ece.getPosColumn(), + ece.getEndPosLine(), + ece.getEndPosColumn()); + ex = ece.getCause(); + } + + return new SqlParseException( + ex.getMessage(), pos, expectedTokenSequences, tokenImage, ex); +} + +/** + * Removes or transforms misleading information from a parse exception. + * + * @param e dirty excn + * + * @return clean excn + */ +JAVACODE ParseException cleanupParseException(ParseException ex) +{ + if (ex.expectedTokenSequences == null) { + return ex; + } + int iIdentifier = Arrays.asList(ex.tokenImage).indexOf(""); + + // Find all sequences in the error which contain identifier. For + // example, + // {} + // {A} + // {B, C} + // {D, } + // {D, A} + // {D, B} + // + // would yield + // {} + // {D} + final List prefixList = new ArrayList(); + for (int i = 0; i < ex.expectedTokenSequences.length; ++i) { + int[] seq = ex.expectedTokenSequences[i]; + int j = seq.length - 1; + int i1 = seq[j]; + if (i1 == iIdentifier) { + int[] prefix = new int[j]; + System.arraycopy(seq, 0, prefix, 0, j); + prefixList.add(prefix); + } + } + + if (prefixList.isEmpty()) { + return ex; + } + + int[][] prefixes = (int[][]) + prefixList.toArray(new int[prefixList.size()][]); + + // Since was one of the possible productions, + // we know that the parser will also have included all + // of the non-reserved keywords (which are treated as + // identifiers in non-keyword contexts). So, now we need + // to clean those out, since they're totally irrelevant. + + final List list = new ArrayList(); + Metadata metadata = getMetadata(); + for (int i = 0; i < ex.expectedTokenSequences.length; ++i) { + int [] seq = ex.expectedTokenSequences[i]; + String tokenImage = ex.tokenImage[seq[seq.length - 1]]; + String token = SqlParserUtil.getTokenVal(tokenImage); + if (token == null || !metadata.isNonReservedKeyword(token)) { + list.add(seq); + continue; + } + boolean match = matchesPrefix(seq, prefixes); + if (!match) { + list.add(seq); + } + } + + ex.expectedTokenSequences = + (int [][]) list.toArray(new int [list.size()][]); + return ex; +} + +JAVACODE boolean matchesPrefix(int[] seq, int[][] prefixes) +{ + nextPrefix: + for (int[] prefix : prefixes) { + if (seq.length == prefix.length + 1) { + for (int k = 0; k < prefix.length; k++) { + if (prefix[k] != seq[k]) { + continue nextPrefix; + } + } + return true; + } + } + return false; +} + +/***************************************** + * Syntactical Descriptions * + *****************************************/ + +SqlNode ExprOrJoinOrOrderedQuery(ExprContext exprContext) : +{ + SqlNode e; + final List list = new ArrayList(); +} +{ + // Lookhead to distinguish between "TABLE emp" (which will be + // matched by ExplicitTable() via Query()) + // and "TABLE fun(args)" (which will be matched by TableRef()) + ( + LOOKAHEAD(2) + e = Query(exprContext) + e = OrderByLimitOpt(e) + { return e; } + | + e = TableRef1(ExprContext.ACCEPT_QUERY_OR_JOIN) + ( e = JoinTable(e) )* + { list.add(e); } + ( AddSetOpQuery(list, exprContext) )* + { return SqlParserUtil.toTree(list); } + ) +} + +/** + * Parses either a row expression or a query expression with an optional + * ORDER BY. + * + *

Postgres syntax for limit: + * + * 

+ *    [ LIMIT { count | ALL } ]
+ *    [ OFFSET start ]
+ *
+ * + *

Trino syntax for limit: + * + * 

+ *    [ OFFSET start ]
+ *    [ LIMIT { count | ALL } ]
+ *
+ * + *

MySQL syntax for limit: + * + * 

+ *    [ LIMIT { count | start, count } ]
+ *
+ * + *

SQL:2008 syntax for limit: + * + * 

+ *    [ OFFSET start { ROW | ROWS } ]
+ *    [ FETCH { FIRST | NEXT } [ count ] { ROW | ROWS } ONLY ]
+ *
+ */ +SqlNode OrderedQueryOrExpr(ExprContext exprContext) : +{ + SqlNode e; +} +{ + e = QueryOrExpr(exprContext) + e = OrderByLimitOpt(e) + { return e; } +} + +/** Reads optional "ORDER BY", "LIMIT", "OFFSET", "FETCH" following a query, + * {@code e}. If any of them are present, adds them to the query; + * otherwise returns the query unchanged. + * Throws if they are present and {@code e} is not a query. */ +SqlNode OrderByLimitOpt(SqlNode e) : +{ + final SqlNodeList orderBy; + final Span s = Span.of(); + SqlNode[] offsetFetch = {null, null}; +} +{ + ( + // use the syntactic type of the expression we just parsed + // to decide whether ORDER BY makes sense + orderBy = OrderBy(e.isA(SqlKind.QUERY)) + | { orderBy = null; } + ) + [ + LimitClause(s, offsetFetch) + [ OffsetClause(s, offsetFetch) ] + | + OffsetClause(s, offsetFetch) + [ + LimitClause(s, offsetFetch) { + if (!this.conformance.isOffsetLimitAllowed()) { + throw SqlUtil.newContextException(s.end(this), + RESOURCE.offsetLimitNotAllowed()); + } + } + | + FetchClause(offsetFetch) + ] + | + FetchClause(offsetFetch) + ] + { + if (orderBy != null || offsetFetch[0] != null || offsetFetch[1] != null) { + return new SqlOrderBy(getPos(), e, + Util.first(orderBy, SqlNodeList.EMPTY), + offsetFetch[0], offsetFetch[1]); + } + return e; + } +} + +/** + * Parses an OFFSET clause in an ORDER BY expression. + */ +void OffsetClause(Span s, SqlNode[] offsetFetch) : +{ +} +{ + // ROW or ROWS is required in SQL:2008 but we make it optional + // because it is not present in Postgres-style syntax. + { s.add(this); } + offsetFetch[0] = UnsignedNumericLiteralOrParam() + [ | ] +} + +/** + * Parses a FETCH clause in an ORDER BY expression. + */ +void FetchClause(SqlNode[] offsetFetch) : +{ +} +{ + // SQL:2008-style syntax. "OFFSET ... FETCH ...". + // If you specify both LIMIT and FETCH, FETCH wins. + ( | ) offsetFetch[1] = UnsignedNumericLiteralOrParam() + ( | ) +} + +/** + * Parses a LIMIT clause in an ORDER BY expression. + */ +void LimitClause(Span s, SqlNode[] offsetFetch) : +{ + final String error; +} +{ + // Postgres-style syntax. "LIMIT ... OFFSET ..." + { s.add(this); } + ( + // MySQL-style syntax. "LIMIT start, count" or "LIMIT start, ALL" + LOOKAHEAD(2) + offsetFetch[0] = UnsignedNumericLiteralOrParam() + + ( + offsetFetch[1] = UnsignedNumericLiteralOrParam() { + error = "count"; + } + | + { + error = "ALL"; + } + ) { + if (!this.conformance.isLimitStartCountAllowed()) { + throw SqlUtil.newContextException(s.end(this), + RESOURCE.limitStartCountOrAllNotAllowed(error)); + } + } + | + offsetFetch[1] = UnsignedNumericLiteralOrParam() + | + + ) +} + +/** + * Parses a leaf in a query expression (SELECT, VALUES or TABLE). + */ +SqlNode LeafQuery(ExprContext exprContext) : +{ + SqlNode e; +} +{ + { + // ensure a query is legal in this context + checkQueryExpression(exprContext); + } + e = SqlSelect() { return e; } +| + e = TableConstructor() { return e; } +| + e = ExplicitTable(getPos()) { return e; } +} + +/** + * Parses a parenthesized query or single row expression. + * Depending on {@code exprContext}, may also accept a join. + */ +SqlNode ParenthesizedExpression(ExprContext exprContext) : +{ + SqlNode e; +} +{ + + { + // we've now seen left paren, so queries inside should + // be allowed as sub-queries + switch (exprContext) { + case ACCEPT_SUB_QUERY: + exprContext = ExprContext.ACCEPT_NONCURSOR; + break; + case ACCEPT_CURSOR: + exprContext = ExprContext.ACCEPT_ALL; + break; + } + } + e = ExprOrJoinOrOrderedQuery(exprContext) + + { + exprContext.throwIfNotCompatible(e); + return e; + } +} + +/** + * Parses a parenthesized query or comma-list of row expressions. + * + *

REVIEW jvs 8-Feb-2004: There's a small hole in this production. It can be + * used to construct something like + * + * 

+ * WHERE x IN (select count(*) from t where c=d,5)
+ *
+ * + *

which should be illegal. The above is interpreted as equivalent to + * + * 

+ * WHERE x IN ((select count(*) from t where c=d),5)
+ *
+ * + *

which is a legal use of a sub-query. The only way to fix the hole is to + * be able to remember whether a subexpression was parenthesized or not, which + * means preserving parentheses in the SqlNode tree. This is probably + * desirable anyway for use in purely syntactic parsing applications (e.g. SQL + * pretty-printer). However, if this is done, it's important to also make + * isA() on the paren node call down to its operand so that we can + * always correctly discriminate a query from a row expression. + */ +SqlNodeList ParenthesizedQueryOrCommaList( + ExprContext exprContext) : +{ + SqlNode e; + final List list = new ArrayList(); + ExprContext firstExprContext = exprContext; + final Span s; +} +{ + + { + // we've now seen left paren, so a query by itself should + // be interpreted as a sub-query + s = span(); + switch (exprContext) { + case ACCEPT_SUB_QUERY: + firstExprContext = ExprContext.ACCEPT_NONCURSOR; + break; + case ACCEPT_CURSOR: + firstExprContext = ExprContext.ACCEPT_ALL; + break; + } + } + e = OrderedQueryOrExpr(firstExprContext) { list.add(e); } + ( + + { + // a comma-list can't appear where only a query is expected + checkNonQueryExpression(exprContext); + } + AddExpression(list, exprContext) + )* + + { + return new SqlNodeList(list, s.end(this)); + } +} + +/** As ParenthesizedQueryOrCommaList, but allows DEFAULT + * in place of any of the expressions. For example, + * {@code (x, DEFAULT, null, DEFAULT)}. */ +SqlNodeList ParenthesizedQueryOrCommaListWithDefault( + ExprContext exprContext) : +{ + SqlNode e; + final List list = new ArrayList(); + ExprContext firstExprContext = exprContext; + final Span s; +} +{ + + { + // we've now seen left paren, so a query by itself should + // be interpreted as a sub-query + s = span(); + switch (exprContext) { + case ACCEPT_SUB_QUERY: + firstExprContext = ExprContext.ACCEPT_NONCURSOR; + break; + case ACCEPT_CURSOR: + firstExprContext = ExprContext.ACCEPT_ALL; + break; + } + } + ( + e = OrderedQueryOrExpr(firstExprContext) { list.add(e); } + | + e = Default() { list.add(e); } + ) + ( + + { + // a comma-list can't appear where only a query is expected + checkNonQueryExpression(exprContext); + } + ( + e = Expression(exprContext) { list.add(e); } + | + e = Default() { list.add(e); } + ) + )* + + { + return new SqlNodeList(list, s.end(this)); + } +} + +/** + * Parses function parameter lists. + * If the list starts with DISTINCT or ALL, it is discarded. + */ +List UnquantifiedFunctionParameterList(ExprContext exprContext) : +{ + final List args; +} +{ + args = FunctionParameterList(exprContext) { + args.remove(0); // remove DISTINCT or ALL, if present + return args; + } +} + +/** + * Parses function parameter lists including DISTINCT keyword recognition, + * DEFAULT, and named argument assignment. + */ +List FunctionParameterList(ExprContext exprContext) : +{ + final SqlLiteral qualifier; + final List list = new ArrayList(); +} +{ + + ( + qualifier = AllOrDistinct() { list.add(qualifier); } + | + { list.add(null); } + ) + AddArg0(list, exprContext) + ( + { + // a comma-list can't appear where only a query is expected + checkNonQueryExpression(exprContext); + } + AddArg(list, exprContext) + )* + + { + return list; + } +} + +SqlLiteral AllOrDistinct() : +{ +} +{ + { return SqlSelectKeyword.DISTINCT.symbol(getPos()); } +| + { return SqlSelectKeyword.ALL.symbol(getPos()); } +} + +void AddArg0(List list, ExprContext exprContext) : +{ + final SqlIdentifier name; + SqlNode e; + final ExprContext firstExprContext; + { + // we've now seen left paren, so queries inside should + // be allowed as sub-queries + switch (exprContext) { + case ACCEPT_SUB_QUERY: + firstExprContext = ExprContext.ACCEPT_NONCURSOR; + break; + case ACCEPT_CURSOR: + firstExprContext = ExprContext.ACCEPT_ALL; + break; + default: + firstExprContext = exprContext; + break; + } + } +} +{ + ( + LOOKAHEAD(2) name = SimpleIdentifier() + | { name = null; } + ) + ( + e = Default() + | + LOOKAHEAD(3) + e = TableParam() + | + e = PartitionedQueryOrQueryOrExpr(firstExprContext) + ) + { + if (name != null) { + e = SqlStdOperatorTable.ARGUMENT_ASSIGNMENT.createCall( + Span.of(name, e).pos(), e, name); + } + list.add(e); + } +} + +void AddArg(List list, ExprContext exprContext) : +{ + final SqlIdentifier name; + SqlNode e; +} +{ + ( + LOOKAHEAD(2) name = SimpleIdentifier() + | { name = null; } + ) + ( + e = Default() + | + e = Expression(exprContext) + | + e = TableParam() + ) + { + if (name != null) { + e = SqlStdOperatorTable.ARGUMENT_ASSIGNMENT.createCall( + Span.of(name, e).pos(), e, name); + } + list.add(e); + } +} + +SqlNode Default() : {} +{ + { + return SqlStdOperatorTable.DEFAULT.createCall(getPos()); + } +} + +/** + * Parses a query (SELECT, UNION, INTERSECT, EXCEPT, VALUES, TABLE) followed by + * the end-of-file symbol. + */ +SqlNode SqlQueryEof() : +{ + SqlNode query; +} +{ + query = OrderedQueryOrExpr(ExprContext.ACCEPT_QUERY) + + { return query; } +} + +/** + * Parses a list of SQL statements separated by semicolon. + * The semicolon is required between statements, but is + * optional at the end. + */ +SqlNodeList SqlStmtList() : +{ + final List stmtList = new ArrayList(); + SqlNode stmt; +} +{ + stmt = SqlStmt() { + stmtList.add(stmt); + } + ( + + [ + stmt = SqlStmt() { + stmtList.add(stmt); + } + ] + )* + + { + return new SqlNodeList(stmtList, Span.of(stmtList).pos()); + } +} + +/** + * Parses an SQL statement. + */ +SqlNode SqlStmt() : +{ + SqlNode stmt; +} +{ + ( +<#-- Add methods to parse additional statements here --> +<#list (parser.statementParserMethods!default.parser.statementParserMethods) as method> + LOOKAHEAD(2) stmt = ${method} + | + + stmt = SqlSetOption(Span.of(), null) + | + stmt = SqlAlter() + | +<#if (parser.createStatementParserMethods!default.parser.createStatementParserMethods)?size != 0> + stmt = SqlCreate() + | + +<#if (parser.dropStatementParserMethods!default.parser.dropStatementParserMethods)?size != 0> + stmt = SqlDrop() + | + +<#if (parser.truncateStatementParserMethods!default.parser.truncateStatementParserMethods)?size != 0> + LOOKAHEAD(2) + stmt = SqlTruncate() + | + + stmt = OrderedQueryOrExpr(ExprContext.ACCEPT_QUERY) + | + stmt = SqlExplain() + | + stmt = SqlDescribe() + | + stmt = SqlInsert() + | + stmt = SqlDelete() + | + stmt = SqlUpdate() + | + stmt = SqlMerge() + | + stmt = SqlProcedureCall() + ) + { + return stmt; + } +} + +/** + * Parses an SQL statement followed by the end-of-file symbol. + */ +SqlNode SqlStmtEof() : +{ + SqlNode stmt; +} +{ + stmt = SqlStmt() + { + return stmt; + } +} + +<#-- Add implementations of additional parser statement calls here --> +<#list (parser.implementationFiles!default.parser.implementationFiles) as file> + <#include "/@includes/"+file /> + + +SqlNodeList ParenthesizedKeyValueOptionCommaList() : +{ + final Span s; + final List list = new ArrayList(); +} +{ + { s = span(); } + + AddKeyValueOption(list) + ( + + AddKeyValueOption(list) + )* + { + return new SqlNodeList(list, s.end(this)); + } +} + +/** +* Parses an option with format key=val whose key is a simple identifier or string literal +* and value is a string literal. +*/ +void AddKeyValueOption(List list) : +{ + final SqlNode key; + final SqlNode value; +} +{ + ( + key = SimpleIdentifier() + | + key = StringLiteral() + ) + + value = StringLiteral() { + list.add(key); + list.add(value); + } +} + +/** Parses an option value (either a string or a numeric) and adds to a list. */ +void AddOptionValue(List list) : +{ + final SqlNode value; +} +{ + ( + value = NumericLiteral() { list.add(value); } + | + value = StringLiteral() { list.add(value); } + ) +} + +/** + * Parses a literal list separated by comma. The literal is either a string or a numeric. + */ +SqlNodeList ParenthesizedLiteralOptionCommaList() : +{ + final Span s; + final List list = new ArrayList(); +} +{ + { s = span(); } + + AddOptionValue(list) ( AddOptionValue(list) )* + { + return new SqlNodeList(list, s.end(this)); + } +} + +void AddHint(List hints) : +{ + final SqlIdentifier hintName; + final SqlNodeList hintOptions; + final SqlHint.HintOptionFormat optionFormat; +} +{ + hintName = SimpleIdentifier() + ( + LOOKAHEAD(5) + hintOptions = ParenthesizedKeyValueOptionCommaList() { + optionFormat = SqlHint.HintOptionFormat.KV_LIST; + } + | + LOOKAHEAD(3) + hintOptions = ParenthesizedSimpleIdentifierList() { + optionFormat = SqlHint.HintOptionFormat.ID_LIST; + } + | + LOOKAHEAD(3) + hintOptions = ParenthesizedLiteralOptionCommaList() { + optionFormat = SqlHint.HintOptionFormat.LITERAL_LIST; + } + | + LOOKAHEAD(2) + [ ] + { + hintOptions = SqlNodeList.EMPTY; + optionFormat = SqlHint.HintOptionFormat.EMPTY; + } + ) + { + hints.add( + new SqlHint(Span.of(hintOptions).end(this), hintName, hintOptions, + optionFormat)); + } +} + +/** Parses hints following a table reference, + * and returns the wrapped table reference. */ +SqlNode TableHints(SqlIdentifier tableName) : +{ + final List hints = new ArrayList(); +} +{ + AddHint(hints) ( AddHint(hints) )* { + final SqlParserPos pos = Span.of(tableName).addAll(hints).end(this); + final SqlNodeList hintList = new SqlNodeList(hints, pos); + return new SqlTableRef(pos, tableName, hintList); + } +} + +/** + * Parses a leaf SELECT expression without ORDER BY. + */ +SqlSelect SqlSelect() : +{ + final List keywords = new ArrayList(); + final SqlLiteral keyword; + final SqlNodeList keywordList; + final List selectList = new ArrayList(); + final SqlNode fromClause; + final SqlNode where; + final SqlNodeList groupBy; + final SqlNode having; + final SqlNodeList windowDecls; + final SqlNode qualify; + final List hints = new ArrayList(); + final Span s; +} +{ +