Skip to content

Commit 2855d10

Browse files
author
Jason Gerlowski
committed
LWSHADOOP-757: Create 7.3.1 SOLR service
This commit creates a SOLR mpack
1 parent dd3074d commit 2855d10

30 files changed

+15452
-0
lines changed

‎build.gradle‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ ext.mpack = 'mpack.json'
1616
ext.repostemplate = 'src/main/template/repos/repoinfo.xml'
1717
ext.reposinfo555 = 'custom-services/SOLR/5.5.5/repos/'
1818
ext.reposinfo662 = 'custom-services/SOLR/6.6.2/repos/'
19+
ext.reposinfo731 = 'custom-services/SOLR/7.3.1/repos/'
1920

2021
distributions {
2122
main {
@@ -41,6 +42,12 @@ distributions {
4142
line.replace("{REPOID}", "${repoid662}")
4243
}
4344
}
45+
into(reposinfo731) {
46+
from { repostemplate }
47+
filter { line ->
48+
line.replace("{REPOID}", "${repoid731}")
49+
}
50+
}
4451
}
4552
}
4653
}

‎gradle.properties‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,7 @@ repoid555=HDP-SOLR-2.6-100
77
# Solr 6.6.2 repo
88
# TODO a Solr repo is needed
99
repoid662=HDP-SOLR-2.6-100
10+
11+
# Solr 7.3.1 repo
12+
# TODO a Solr repo is needed
13+
repoid731=HDP-SOLR-2.6-100

‎src/main/mpack/common-services/SOLR/7.3.1/alerts.json‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
{
2+
"SOLR": {
3+
"SOLR_SERVER": [
4+
{
5+
"name": "solr",
6+
"label": "Solr Web UI",
7+
"description": "This host-level alert is triggered if the Solr Cloud Instance is unreachable.",
8+
"interval": 1,
9+
"scope": "ANY",
10+
"source": {
11+
"type": "WEB",
12+
"uri": {
13+
"http": "{{solr-config-env/solr_config_port}}",
14+
"https": "{{solr-config-env/solr_config_port}}",
15+
"https_property": "{{solr-ssl/solr_ssl_enable}}",
16+
"https_property_value": "true",
17+
"kerberos_keytab": "{{solr-config-env/solr_spnego_keytab_path}}",
18+
"kerberos_principal": "{{solr-config-env/solr_spnego_principal_name}}",
19+
"connection_timeout": 5.0,
20+
"default_port": 8983
21+
},
22+
"reporting": {
23+
"ok": {
24+
"text": "HTTP {0} response in {2:.3f}s"
25+
},
26+
"warning": {
27+
"text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
28+
},
29+
"critical": {
30+
"text": "Connection failed to {1} ({3})"
31+
}
32+
}
33+
}
34+
},
35+
{
36+
"name": "solr_cpu_usage",
37+
"label": "Solr CPU Utilization",
38+
"description": "This host-level alert is triggered if CPU utilization of the Solr exceeds certain warning and critical thresholds. It checks the Solr JMX Servlet for the SystemCPULoad property. The threshold values are in percent.",
39+
"interval": 5,
40+
"scope": "ANY",
41+
"source": {
42+
"type": "SCRIPT",
43+
"path": "SOLR/7.3.1/package/alerts/alert_solr_cpu_metrics.py",
44+
"parameters": [
45+
{
46+
"name": "connection.timeout",
47+
"value": 5.0,
48+
"type": "NUMERIC",
49+
"units": "seconds",
50+
"visibility": "HIDDEN"
51+
},
52+
{
53+
"name": "app.id",
54+
"value": "solr-host-app",
55+
"type": "STRING",
56+
"visibility": "HIDDEN"
57+
},
58+
{
59+
"name": "metric.names",
60+
"value": "solr.jvm.gauge.os.processCpuLoad",
61+
"type": "STRING",
62+
"visibility": "HIDDEN"
63+
},
64+
{
65+
"name": "metric.solr.cpu.warning.threshold",
66+
"display_name": "Warning threshold",
67+
"type": "PERCENT",
68+
"units": "%",
69+
"value": 75,
70+
"description": "CPU usage is greater than 50%",
71+
"threshold": "WARNING"
72+
},
73+
{
74+
"name": "metric.solr.cpu.critical.threshold",
75+
"display_name": "Critical threshold",
76+
"type": "PERCENT",
77+
"units": "%",
78+
"value": 90,
79+
"description": "CPU usage is greater than 75%",
80+
"threshold": "CRITICAL"
81+
}
82+
]
83+
}
84+
},
85+
{
86+
"name": "solr_memory",
87+
"label": "Solr Memory Utilization",
88+
"description": "This host-level alert is triggered if CPU utilization of the Solr exceeds certain warning and critical thresholds. It checks the Solr JMX Servlet for the SystemCPULoad property. The threshold values are in percent.",
89+
"interval": 5,
90+
"scope": "ANY",
91+
"source": {
92+
"type": "SCRIPT",
93+
"path": "SOLR/7.3.1/package/alerts/alert_solr_memory_metrics.py",
94+
"parameters": [
95+
{
96+
"name": "connection.timeout",
97+
"value": 5.0,
98+
"type": "NUMERIC",
99+
"units": "seconds",
100+
"visibility": "HIDDEN"
101+
},
102+
{
103+
"name": "app.id",
104+
"value": "solr-host-app",
105+
"type": "STRING",
106+
"visibility": "HIDDEN"
107+
},
108+
{
109+
"name": "metric.names",
110+
"value": "solr.jvm.gauge.memory.total.used,solr.jvm.gauge.memory.total.max",
111+
"type": "STRING",
112+
"visibility": "HIDDEN"
113+
},
114+
{
115+
"name": "metric.solr.memory.warning.threshold",
116+
"display_name": "Warning threshold",
117+
"type": "PERCENT",
118+
"units": "%",
119+
"value": 75,
120+
"description": "Memory usage is greater than 50%",
121+
"threshold": "WARNING"
122+
},
123+
{
124+
"name": "metric.solr.memory.critical.threshold",
125+
"display_name": "Critical threshold",
126+
"type": "PERCENT",
127+
"units": "%",
128+
"value": 90,
129+
"description": "Memory usage is greater than 75%",
130+
"threshold": "CRITICAL"
131+
}
132+
]
133+
}
134+
}
135+
]
136+
}
137+
}

‎src/main/mpack/common-services/SOLR/7.3.1/configuration/solr-config-env.xml‎

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
<?xml version="1.0"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3+
4+
<configuration supports_final="false" supports_adding_forbidden="true">
5+
6+
<property>
7+
<name>solr_stop_wait</name>
8+
<value>180</value>
9+
<description>This controls the number of seconds that the solr script will wait for Solr to stop gracefully
10+
or Solr to start. If the graceful stop fails, the script will forcibly stop Solr. If the start fails,
11+
the script will give up waiting and display the last few lines of the logfile.
12+
</description>
13+
<value-attributes>
14+
<type>int</type>
15+
</value-attributes>
16+
<display-name>Solr stop wait</display-name>
17+
</property>
18+
19+
20+
<!-- solr.in.sh -->
21+
<property>
22+
<name>solr_in_sh_template</name>
23+
<display-name>solr.in.sh template</display-name>
24+
<description>This is a template for the solr.in.sh file</description>
25+
<value>
26+
# Licensed to the Apache Software Foundation (ASF) under one or more
27+
# contributor license agreements. See the NOTICE file distributed with
28+
# this work for additional information regarding copyright ownership.
29+
# The ASF licenses this file to You under the Apache License, Version 2.0
30+
# (the "License"); you may not use this file except in compliance with
31+
# the License. You may obtain a copy of the License at
32+
#
33+
# http://www.apache.org/licenses/LICENSE-2.0
34+
#
35+
# Unless required by applicable law or agreed to in writing, software
36+
# distributed under the License is distributed on an "AS IS" BASIS,
37+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
38+
# See the License for the specific language governing permissions and
39+
# limitations under the License.
40+
41+
# Settings here will override settings in existing env vars or in bin/solr. The default shipped state
42+
# of this file is completely commented.
43+
44+
# By default the script will use JAVA_HOME to determine which java
45+
# to use, but you can set a specific path for Solr to use without
46+
# affecting other Java applications on your server/workstation.
47+
SOLR_JAVA_HOME={{java64_home}}
48+
49+
# This controls the number of seconds that the solr script will wait for
50+
# Solr to stop gracefully or Solr to start. If the graceful stop fails,
51+
# the script will forcibly stop Solr. If the start fails, the script will
52+
# give up waiting and display the last few lines of the logfile.
53+
SOLR_STOP_WAIT="{{solr_stop_wait}}"
54+
55+
# Increase Java Heap as needed to support your indexing / query needs
56+
SOLR_HEAP="{{solr_config_memory}}"
57+
58+
# Expert: If you want finer control over memory options, specify them directly
59+
# Comment out SOLR_HEAP if you are using this though, that takes precedence
60+
#SOLR_JAVA_MEM="-Xms512m -Xmx512m"
61+
62+
# Enable verbose GC logging...
63+
# * If this is unset, various default options will be selected depending on which JVM version is in use
64+
# * For Java 8: if this is set, additional params will be added to specify the log file and rotation
65+
# * For Java 9 or higher: each included opt param that starts with '-Xlog:gc', but does not include an
66+
# output specifier, will have a 'file' output specifier (as well as formatting and rollover options)
67+
# appended, using the effective value of the SOLR_LOGS_DIR.
68+
#
69+
#GC_LOG_OPTS='-Xlog:gc*' # (Java 9+)
70+
#GC_LOG_OPTS="-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \
71+
# -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime"
72+
73+
# These GC settings have shown to work well for a number of common Solr workloads
74+
{{solr_hdfs_prefix}}GC_TUNE="$GC_TUNE -XX:MaxDirectMemorySize=20g -XX:+UseLargePages"
75+
76+
# Set the ZooKeeper connection string if using an external ZooKeeper ensemble
77+
# e.g. host1:2181,host2:2181/chroot
78+
# Leave empty if not using SolrCloud
79+
{{solr_cloud_mode_prefix}}ZK_HOST="{{zookeeper_hosts}}{{solr_cloud_zk_directory}}"
80+
81+
# Set the ZooKeeper client timeout (for SolrCloud mode)
82+
#ZK_CLIENT_TIMEOUT="15000"
83+
84+
# By default the start script uses "localhost"; override the hostname here
85+
# for production SolrCloud environments to control the hostname exposed to cluster state
86+
SOLR_HOST="{{solr_hostname}}"
87+
88+
# By default the start script uses UTC; override the timezone if needed
89+
#SOLR_TIMEZONE="UTC"
90+
91+
# Set to true to activate the JMX RMI connector to allow remote JMX client applications
92+
# to monitor the JVM hosting Solr; set to "false" to disable that behavior
93+
# (false is recommended in production environments)
94+
#ENABLE_REMOTE_JMX_OPTS="false"
95+
96+
# The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here
97+
# RMI_PORT=18983
98+
99+
# Anything you add to the SOLR_OPTS variable will be included in the java
100+
# start command line as-is, in ADDITION to other options. If you specify the
101+
# -a option on start script, those options will be appended as well. Examples:
102+
#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=3000"
103+
#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoCommit.maxTime=60000"
104+
#SOLR_OPTS="$SOLR_OPTS -Dsolr.clustering.enabled=true"
105+
106+
# Not cloud + kerberos start settings
107+
{{solr_not_cloud_mode_prefix}}{{solr_kerberos_prefix}}SOLR_OPTS="$SOLR_OPTS \
108+
{{solr_not_cloud_mode_prefix}}{{solr_kerberos_prefix}}-DauthenticationPlugin=org.apache.solr.security.KerberosPlugin"
109+
110+
# HDFS start settings
111+
{{solr_hdfs_prefix}}SOLR_OPTS="$SOLR_OPTS -Dsolr.directoryFactory=HdfsDirectoryFactory \
112+
{{solr_hdfs_prefix}} -Dsolr.lock.type=hdfs -Dsolr.hdfs.home={{default_fs}}{{solr_hdfs_directory}} \
113+
{{solr_hdfs_prefix}} -Dsolr.hdfs.confdir={{hadoop_conf_dir}}"
114+
115+
# Kerberos + HDFS start settings
116+
{{solr_hdfs_prefix}}{{solr_kerberos_prefix}}SOLR_OPTS="$SOLR_OPTS \
117+
{{solr_hdfs_prefix}}{{solr_kerberos_prefix}}-Dsolr.hdfs.security.kerberos.enabled=true \
118+
{{solr_hdfs_prefix}}{{solr_kerberos_prefix}}-Dsolr.hdfs.security.kerberos.keytabfile={{solr_kerberos_keytab}} \
119+
{{solr_hdfs_prefix}}{{solr_kerberos_prefix}}-Dsolr.hdfs.security.kerberos.principal={{solr_kerberos_principal}}"
120+
121+
# Location where the bin/solr script will save PID files for running instances
122+
# If not set, the script will create PID files in $SOLR_TIP/bin
123+
SOLR_PID_DIR={{solr_config_pid_dir}}
124+
125+
# Path to a directory for Solr to store cores and their data. By default, Solr will use server/solr
126+
# If solr.xml is not stored in ZooKeeper, this directory needs to contain solr.xml
127+
SOLR_HOME={{solr_config_data_dir}}
128+
129+
# Solr provides a default Log4J configuration properties file in server/resources
130+
# however, you may want to customize the log settings and file appender location
131+
# so you can point the script to use a different log4j.properties file
132+
LOG4J_PROPS={{solr_config_conf_dir}}/log4j.properties
133+
134+
# Changes the logging level. Valid values: ALL, TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF. Default is INFO
135+
# This is an alternative to changing the rootLogger in log4j.properties
136+
#SOLR_LOG_LEVEL=INFO
137+
138+
# Location where Solr should write logs to. Absolute or relative to solr start dir
139+
SOLR_LOGS_DIR={{solr_config_log_dir}}
140+
141+
# Enables log rotation, cleanup, and archiving during start. Setting SOLR_LOG_PRESTART_ROTATION=false will skip start
142+
# time rotation of logs, and the archiving of the last GC and console log files. It does not affect Log4j configuration.
143+
# This pre-startup rotation may need to be disabled depending how much you customize the default logging setup.
144+
#SOLR_LOG_PRESTART_ROTATION=true
145+
146+
# Sets the port Solr binds to, default is 8983
147+
SOLR_PORT={{solr_config_port}}
148+
149+
# Uncomment to set SSL-related system properties
150+
# Be sure to update the paths to the correct keystore for your environment
151+
{{solr_ssl_prefix}}SOLR_SSL_KEY_STORE={{ solr_ssl_key_store }}
152+
{{solr_ssl_prefix}}SOLR_SSL_KEY_STORE_PASSWORD={{ solr_ssl_key_store_password }}
153+
{{solr_ssl_prefix}}SOLR_SSL_KEY_STORE_TYPE={{solr_ssl_key_store_type}}
154+
{{solr_ssl_prefix}}SOLR_SSL_TRUST_STORE={{ solr_ssl_trust_store }}
155+
{{solr_ssl_prefix}}SOLR_SSL_TRUST_STORE_PASSWORD={{ solr_ssl_trust_store_password }}
156+
{{solr_ssl_prefix}}SOLR_SSL_TRUST_STORE_TYPE={{solr_ssl_trust_store_type}}
157+
{{solr_ssl_prefix}}SOLR_SSL_NEED_CLIENT_AUTH={{ solr_ssl_need_client_auth }}
158+
{{solr_ssl_prefix}}SOLR_SSL_WANT_CLIENT_AUTH={{ solr_ssl_want_client_auth }}
159+
160+
# Uncomment if you want to override previously defined SSL values for HTTP client
161+
# otherwise keep them commented and the above values will automatically be set for HTTP clients
162+
#SOLR_SSL_CLIENT_KEY_STORE=
163+
#SOLR_SSL_CLIENT_KEY_STORE_PASSWORD=
164+
#SOLR_SSL_CLIENT_KEY_STORE_TYPE=
165+
#SOLR_SSL_CLIENT_TRUST_STORE=
166+
#SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=
167+
#SOLR_SSL_CLIENT_TRUST_STORE_TYPE=
168+
169+
# Settings for authentication
170+
# Please configure only one of SOLR_AUTHENTICATION_CLIENT_CONFIGURER or SOLR_AUTH_TYPE parameters
171+
{{solr_kerberos_prefix}}SOLR_AUTH_TYPE="kerberos"
172+
{{solr_kerberos_prefix}}SOLR_AUTHENTICATION_OPTS="-Djava.security.auth.login.config={{solr_kerberos_jaas_config}} \
173+
{{solr_kerberos_prefix}}-Dsolr.kerberos.cookie.domain={{solr_kerberos_cookie_domain}} \
174+
{{solr_kerberos_prefix}}-Dsolr.kerberos.cookie.portaware=true \
175+
{{solr_kerberos_prefix}}-Dsolr.kerberos.principal={{solr_spnego_principal}} \
176+
{{solr_kerberos_prefix}}-Dsolr.kerberos.keytab={{solr_spnego_keytab}}"
177+
178+
# Settings for ZK ACL
179+
#SOLR_ZK_CREDS_AND_ACLS="-DzkACLProvider=org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider \
180+
# -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider \
181+
# -DzkDigestUsername=admin-user -DzkDigestPassword=CHANGEME-ADMIN-PASSWORD \
182+
# -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD"
183+
#SOLR_OPTS="$SOLR_OPTS $SOLR_ZK_CREDS_AND_ACLS"
184+
</value>
185+
<value-attributes>
186+
<type>content</type>
187+
</value-attributes>
188+
</property>
189+
190+
191+
<property>
192+
<name>solr_xml_template</name>
193+
<display-name>solr.xml template</display-name>
194+
<description>This is a template for the solr.xml file</description>
195+
<value/>
196+
<property-type>VALUE_FROM_PROPERTY_FILE</property-type>
197+
<value-attributes>
198+
<property-file-name>solr.xml.j2</property-file-name>
199+
<property-file-type>xml</property-file-type>
200+
</value-attributes>
201+
</property>
202+
</configuration>

0 commit comments

Comments
 (0)