Skip to content

Commit e203db2

Browse files
weiqingysrinipunuru
weiqingy
authored and
srinipunuru
committed
SAMZA-1901: Implementation of Samza SQL Shell
## What changes were proposed in this pull request? This PR is to implement Samza SQL shell. The document about the shell was attached [here](https://issues.apache.org/jira/browse/SAMZA-1901). ## How was this patch tested? 1. Add unit tests 2. Run the shell with use cases mentioned in the attached document under https://issues.apache.org/jira/browse/SAMZA-1901 Author: Weiqing Yang <yangweiqing001@gmail.com> Reviewers: Srinivasulu Punuru <spunuru@linkedin.com>, Aditya Toomula <atoomula@linkedin.com> Closes apache#654 from weiqingy/samza-shell
1 parent fd6f1ed commit e203db2

38 files changed

+3764
-5
lines changed

‎build.gradle‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,34 @@ project(':samza-sql') {
336336
}
337337
}
338338

339+
project(':samza-sql-shell') {
340+
apply plugin: 'java'
341+
342+
dependencies {
343+
compile project(':samza-sql')
344+
compile project(':samza-tools')
345+
compile project(":samza-core_$scalaVersion")
346+
compile project(':samza-api')
347+
compile project(":samza-kafka_$scalaVersion")
348+
compile project(':samza-azure')
349+
compile "net.java.dev.jna:jna:$jnaVersion"
350+
compile "org.jline:jline:$jlineVersion"
351+
352+
testCompile "junit:junit:$junitVersion"
353+
}
354+
355+
tasks.create(name: "releaseSqlShellTarGz", dependsOn: configurations.archives.artifacts, type: Tar) {
356+
into "samza-sql-shell-${version}"
357+
compression = Compression.GZIP
358+
from(project.file("./scripts")) { into "scripts/" }
359+
from(project.file("./conf")) { into "conf/" }
360+
from(project(':samza-shell').file("src/main/bash/run-class.sh")) { into "scripts/" }
361+
from(configurations.runtime) { into("lib/") }
362+
from(configurations.archives.artifacts.files) { into("lib/") }
363+
duplicatesStrategy 'exclude'
364+
}
365+
}
366+
339367
project(':samza-tools') {
340368
apply plugin: 'java'
341369

‎gradle/dependency-versions.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,6 @@
4747
zkClientVersion = "0.8"
4848
zookeeperVersion = "3.4.6"
4949
failsafeVersion = "1.1.0"
50+
jlineVersion = "3.8.2"
51+
jnaVersion = "4.5.1"
5052
}

‎samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,10 +70,8 @@
7070
static Config mergeConfig(Map<String, String> originalConfig, Map<String, String> generatedConfig) {
7171
Map<String, String> mergedConfig = new HashMap<>(generatedConfig);
7272
originalConfig.forEach((k, v) -> {
73-
if (generatedConfig.containsKey(k) &&
74-
!Objects.equals(generatedConfig.get(k), v)) {
75-
LOG.info("Replacing generated config for key: {} value: {} with original config value: {}",
76-
k, generatedConfig.get(k), v);
73+
if (generatedConfig.containsKey(k) && !Objects.equals(generatedConfig.get(k), v)) {
74+
LOG.info("Replacing generated config for key: {} value: {} with original config value: {}", k, generatedConfig.get(k), v);
7775
}
7876
mergedConfig.put(k, v);
7977
});

‎samza-sql-shell/conf/samza-sql-shell-log4j.xml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
<?xml version="1.0" encoding="UTF-8" ?>
2+
<!--
3+
4+
Licensed to the Apache Software Foundation (ASF) under one
5+
or more contributor license agreements. See the NOTICE file
6+
distributed with this work for additional information
7+
regarding copyright ownership. The ASF licenses this file
8+
to you under the Apache License, Version 2.0 (the
9+
"License"); you may not use this file except in compliance
10+
with the License. You may obtain a copy of the License at
11+
12+
http://www.apache.org/licenses/LICENSE-2.0
13+
14+
Unless required by applicable law or agreed to in writing,
15+
software distributed under the License is distributed on an
16+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
KIND, either express or implied. See the License for the
18+
specific language governing permissions and limitations
19+
under the License.
20+
21+
-->
22+
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
23+
24+
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
25+
26+
<appender name="console" class="org.apache.log4j.ConsoleAppender">
27+
<layout class="org.apache.log4j.PatternLayout">
28+
<param name="ConversionPattern"
29+
value="%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n" />
30+
</layout>
31+
</appender>
32+
33+
<appender name="file" class="org.apache.log4j.RollingFileAppender">
34+
<param name="append" value="false" />
35+
<param name="maxFileSize" value="10MB" />
36+
<param name="maxBackupIndex" value="10" />
37+
<param name="file" value="${LOG_HOME}/logs/samza-sql-shell.log" />
38+
<layout class="org.apache.log4j.PatternLayout">
39+
<param name="ConversionPattern"
40+
value="%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n" />
41+
</layout>
42+
</appender>
43+
44+
<root>
45+
<level value="info" />
46+
<appender-ref ref="file" />
47+
</root>
48+
</log4j:configuration>
49+

‎samza-sql-shell/conf/shell-defaults.conf‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
19+
# Default system properties included when running samza-sql-shell.
20+
# This is useful for setting default environmental settings.
21+
22+
# Example:
23+
shell.executor = org.apache.samza.sql.client.impl.SamzaExecutor
24+
samza.sql.output = compact
25+
# samza.sql.system.kafka.address = localhost:2181
26+
# samza.sql.relSchemaProvider.config.schemaDir = /tmp/schemas/
27+
# samza.sql.ioResolver = config
28+
# samza.sql.udfResolver = config

‎samza-sql-shell/scripts/samza-sql-shell.sh‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
if [ `uname` == 'Linux' ];
20+
then
21+
base_dir=$(readlink -f $(dirname $0))
22+
else
23+
base_dir=$(dirname $0)
24+
fi
25+
26+
parent_dir="$(dirname "$base_dir")"
27+
28+
CONF_FILE="-conf $parent_dir/conf/shell-defaults.conf"
29+
30+
if [ "x$LOG_HOME" = "x" ]; then
31+
export LOG_HOME=$parent_dir
32+
fi
33+
34+
if [ "x$LOG4J_OPTS" = "x" ]; then
35+
export LOG4J_OPTS="-Dlog4j.configuration=file:$parent_dir/conf/samza-sql-shell-log4j.xml"
36+
fi
37+
38+
if [ "x$HEAP_OPTS" = "x" ]; then
39+
export HEAP_OPTS="-Xmx4G -Xms4G"
40+
fi
41+
42+
exec $base_dir/run-class.sh $LOG4J_OPTS -DLOG_HOME=$LOG_HOME org.apache.samza.sql.client.cli.Main $CONF_FILE "$@"

‎samza-sql-shell/src/main/java/org/apache/samza/sql/client/cli/CliCommand.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.samza.sql.client.cli;
21+
22+
/**
23+
* A shell command containing command name and parameters.
24+
*/
25+
class CliCommand {
26+
private CliCommandType commandType;
27+
private String parameters;
28+
29+
public CliCommand(CliCommandType cmdType) {
30+
this.commandType = cmdType;
31+
}
32+
33+
public CliCommand(CliCommandType cmdType, String parameters) {
34+
this(cmdType);
35+
this.parameters = parameters;
36+
}
37+
38+
public CliCommandType getCommandType() {
39+
return commandType;
40+
}
41+
42+
public String getParameters() {
43+
return parameters;
44+
}
45+
46+
public void setParameters(String parameters) {
47+
this.parameters = parameters;
48+
}
49+
50+
public String getFullCommand() {
51+
return commandType.getCommandName() + CliConstants.SPACE + parameters;
52+
}
53+
}

‎samza-sql-shell/src/main/java/org/apache/samza/sql/client/cli/CliCommandType.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.samza.sql.client.cli;
21+
22+
import java.util.ArrayList;
23+
import java.util.List;
24+
25+
/**
26+
* Enum all the commands we now support along with descriptions.
27+
*/
28+
enum CliCommandType {
29+
SHOW_TABLES("SHOW TABLES", "Shows all available tables.", "Usage: SHOW TABLES <table name>"),
30+
SHOW_FUNCTIONS("SHOW FUNCTIONS", "Shows all available UDFs.", "SHOW FUNCTION"),
31+
DESCRIBE("DESCRIBE", "Describes a table.", "Usage: DESCRIBE <table name>"),
32+
33+
SELECT("SELECT", "\tExecutes a SQL SELECT query.", "SELECT uses a standard streaming SQL syntax."),
34+
EXECUTE("EXECUTE", "\tExecute a sql file.", "EXECUTE <URI of a sql file>"),
35+
INSERT_INTO("INSERT INTO", "Executes a SQL INSERT INTO.", "INSERT INTO uses a standard streaming SQL syntax."),
36+
LS("LS", "\tLists all background executions.", "LS [execution ID]"),
37+
STOP("STOP", "\tStops an execution.", "Usage: STOP <execution ID>"),
38+
RM("RM", "\tRemoves an execution from the list.", "Usage: RM <execution ID>"),
39+
40+
HELP("HELP", "\tDisplays this help message.", "Usage: HELP [command]"),
41+
SET("SET", "\tSets a variable.", "Usage: SET VAR=VAL"),
42+
CLEAR("CLEAR", "\tClears the screen.", "CLEAR"),
43+
EXIT("EXIT", "\tExits the shell.", "Exit"),
44+
QUIT("QUIT", "\tQuits the shell.", "QUIT"),
45+
46+
INVALID_COMMAND("INVALID_COMMAND", "INVALID_COMMAND", "INVALID_COMMAND");
47+
48+
private final String cmdName;
49+
private final String description;
50+
private final String usage;
51+
52+
CliCommandType(String cmdName, String description, String usage) {
53+
this.cmdName = cmdName;
54+
this.description = description;
55+
this.usage = usage;
56+
}
57+
58+
public static List<String> getAllCommands() {
59+
List<String> cmds = new ArrayList<String>();
60+
for (CliCommandType t : CliCommandType.values()) {
61+
if (t != INVALID_COMMAND)
62+
cmds.add(t.getCommandName());
63+
}
64+
return cmds;
65+
}
66+
67+
public String getCommandName() {
68+
return cmdName;
69+
}
70+
71+
public String getDescription() {
72+
return description;
73+
}
74+
75+
public String getUsage() {
76+
return usage;
77+
}
78+
}

‎samza-sql-shell/src/main/java/org/apache/samza/sql/client/cli/CliConstants.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.samza.sql.client.cli;
21+
22+
/**
23+
* Constant definitions for the shell.
24+
*/
25+
class CliConstants {
26+
public static final String APP_NAME = "Samza SQL Shell";
27+
public static final String WINDOW_TITLE = "Samza SQL Shell";
28+
public static final String PROMPT_1ST = "Samza SQL";
29+
public static final String PROMPT_1ST_END = "> ";
30+
31+
// All shell environment variables starts with the prefix
32+
public static final String CONFIG_SHELL_PREFIX = "shell.";
33+
// Specifies the executor used by the shell
34+
public static final String CONFIG_EXECUTOR = "shell.executor";
35+
36+
public static final String VERSION = "0.0.1";
37+
38+
39+
public static final String WELCOME_MESSAGE;
40+
static {
41+
WELCOME_MESSAGE =
42+
" ___ ___ ___ ___ ___ \n" +
43+
" / /\\ / /\\ / /\\ /__/\\ / /\\ \n" +
44+
" / /::\\ / /::\\ / /::| \\ \\:\\ / /::\\ \n"+
45+
" /__/:/\\:\\ / /:/\\:\\ / /:|:| \\ \\:\\ / /:/\\:\\ \n"+
46+
" _\\_ \\:\\ \\:\\ / /::\\ \\:\\ / /:/|:|__ \\ \\:\\ / /::\\ \\:\\ \n"+
47+
" /__/\\ \\:\\ \\:\\ /__/:/\\:\\_\\:\\ /__/:/_|::::\\ ______\\__\\:\\ /__/:/\\:\\_\\:\\ \n"+
48+
" \\ \\:\\ \\:\\_\\/ \\__\\/ \\:\\/:/ \\__\\/ /~~/:/ \\ \\::::::::/ \\__\\/ \\:\\/:/ \n"+
49+
" \\ \\:\\_\\:\\ \\__\\::/ / /:/ \\ \\:\\~~~~~ \\__\\::/ \n"+
50+
" \\ \\:\\/:/ / /:/ / /:/ \\ \\:\\ / /:/ \n"+
51+
" \\ \\::/ /__/:/ /__/:/ \\ \\:\\ /__/:/ \n"+
52+
" \\__\\/ \\__\\/ \\__\\/ \\__\\/ \\__\\/ \n\n"+
53+
"Welcome to Samza SQL shell (V" + VERSION + "). Enter HELP for all commands.\n\n";
54+
}
55+
56+
public static final char SPACE = '\u0020';
57+
}

0 commit comments

Comments
 (0)