LWSHADOOP-487: Move Solr stack to AMP model (#1)

* Move Solr stack to AMP model
lucidworks · Jun 7, 2016 · 608724c · 608724c
1 parent 228c8df
commit 608724c
Show file tree

Hide file tree

Showing 57 changed files with 1,834 additions and 1,089 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,7 @@
 .DS_Store
 .class
 .swp
+.idea
+*.iml
+.gradle/
+build/
diff --git a/README.md b/README.md
@@ -1,96 +1,21 @@
-#### An Ambari Stack for Solr
-Ambari stack for easily installing and managing Solr on HDP cluster
+## Solr service for Ambari
 
-- Download HDP 2.2 sandbox VM image (Sandbox_HDP_2.2_VMware.ova) from [Hortonworks website](http://hortonworks.com/products/hortonworks-sandbox/)
-- Import Sandbox_HDP_2.2_VMware.ova into VMWare and set the VM memory size to 8GB
-- Now start the VM
-- After it boots up, find the IP address of the VM and add an entry into your machines hosts file e.g.
-```
-192.168.191.241 sandbox.hortonworks.com sandbox    
-```
-- Connect to the VM via SSH (password hadoop) and start Ambari server
-```
-ssh root@sandbox.hortonworks.com
-/root/start_ambari.sh
-```
-- If you are running on sandbox, Solr is already installed under /opt/solr. You can either rename this dir or install the service to different dir 
-```
-mv /opt/solr /opt/solr-orig
-```
-- To deploy the Solr stack, run below
-```
-cd /var/lib/ambari-server/resources/stacks/HDP/2.2/services
-git clone https://github.com/abajwa-hw/solr-stack.git   
-```
+Stop Ambari server:
 
-- Restart Ambari
-```
-#on sandbox
-sudo service ambari restart
+    ambari-server stop
 
-#on non-sandbox
-sudo service ambari-server restart
+Download solr-stack repository:
 
-```
-- Then you can click on 'Add Service' from the 'Actions' dropdown menu in the bottom left of the Ambari dashboard:
+    git clone https://github.com/lucidworks/solr-stack
 
-On bottom left -> Actions -> Add service -> check Solr service -> Next -> Next -> Next -> Deploy
+Create Solr service pack:
 
-On the configuration page, please ensure that you point mvn.dir property to the full path to mvn executable e.g. /usr/bin/mvn
-Also ensure that the install location you are choosing (/opt/solr by default) does not exist
+    ./gradlew clean makePackage
 
-- On successful deployment you will see the Solr service as part of Ambari stack and will be able to start/stop the service from here:
-![Image](../master/screenshots/1.png?raw=true)
+Deploy the Solr service on Ambari server:
 
-- You can see the parameters you configured under 'Configs' tab
-![Image](../master/screenshots/2.png?raw=true)
+    ambari-server install-mpack --mpack=build/solr-service-mpack-{serviceVersion}.tar.gz -v
 
+Start Ambari server:
 
-#### Use Solr 
-
-- Lauch the Solr webapp via navigating to http://sandbox.hortonworks.com:8983/
-
-- Alternatively, you can launch it from Ambari via [iFrame view](https://github.com/abajwa-hw/iframe-view)
-![Image](../master/screenshots/3.png?raw=true)
-
-
-
-- One benefit to wrapping the component in Ambari service is that you can now monitor/manage this service remotely via REST API
-```
-export SERVICE=SOLR
-export PASSWORD=admin
-export AMBARI_HOST=sandbox.hortonworks.com
-export CLUSTER=Sandbox
-
-#get service status
-curl -u admin:$PASSWORD -i -H 'X-Requested-By: ambari' -X GET http://$AMBARI_HOST:8080/api/v1/clusters/$CLUSTER/services/$SERVICE
-
-#start service
-curl -u admin:$PASSWORD -i -H 'X-Requested-By: ambari' -X PUT -d '{"RequestInfo": {"context" :"Start $SERVICE via REST"}, "Body": {"ServiceInfo": {"state": "STARTED"}}}' http://$AMBARI_HOST:8080/api/v1/clusters/$CLUSTER/services/$SERVICE
-
-#stop service
-curl -u admin:$PASSWORD -i -H 'X-Requested-By: ambari' -X PUT -d '{"RequestInfo": {"context" :"Stop $SERVICE via REST"}, "Body": {"ServiceInfo": {"state": "INSTALLED"}}}' http://$AMBARI_HOST:8080/api/v1/clusters/$CLUSTER/services/$SERVICE
-```
-#### Remove Solr service
-
-- To remove the Solr service: 
-  - Stop the service via Ambari
-  - Delete the service
-
-```
-export SERVICE=SOLR
-export PASSWORD=admin
-export AMBARI_HOST=sandbox.hortonworks.com
-export CLUSTER=Sandbox    
-curl -u admin:$PASSWORD -i -H 'X-Requested-By: ambari' -X DELETE http://$AMBARI_HOST:8080/api/v1/clusters/$CLUSTER/services/$SERVICE
-```
-  - Remove artifacts 
-
-    ```
-    rm -rf /var/lib/ambari-server/resources/stacks/HDP/2.2/services/solr-stack
-    rm -rf /opt/solr
-    ```
-  - Restart Ambari
-    ```
-    service ambari restart
-    ```
+    ambari-server start
diff --git a/build.gradle b/build.gradle
@@ -0,0 +1,76 @@
+import groovy.json.JsonSlurper
+
+buildscript {
+    repositories {
+        mavenCentral()
+    }
+}
+
+plugins {
+    id 'base'
+    id 'idea'
+}
+
+task wrapper(type: Wrapper) {
+    distributionUrl = "https://services.gradle.org/distributions/gradle-${gradleVersion}-bin.zip"
+}
+
+task setPackageVersion << {
+    ext.parsePackageVersion = { template, output, templates ->
+        def templateFile = new File(template)
+        def parsedContent = templateFile.text
+        templates.each { pattern, value ->
+            parsedContent = parsedContent.replaceAll(pattern, value)
+        }
+        def outputFile = new File(output)
+        outputFile.text = parsedContent
+    }
+
+    if (!buildDir.exists()) {
+        new File("${buildDir}/tmp/common-services").mkdirs()
+        new File("${buildDir}/tmp/custom-services").mkdirs()
+    }
+
+    parsePackageVersion("mpack.json.template", "${buildDir}/tmp/mpack.json", ["\\{serviceVersion\\}"  : serviceVersion,
+                                                                            "\\{minAmbariVersion\\}": minAmbariVersion,
+                                                                            "\\{stackVersion\\}"    : stackVersion])
+    parsePackageVersion("common-services/SOLR/serviceVersion/metainfo.xml.template",
+            "${buildDir}/tmp/common-services/metainfo.xml", ["\\{serviceVersion\\}": serviceVersion])
+    parsePackageVersion("custom-services/SOLR/serviceVersion/metainfo.xml.template",
+            "${buildDir}/tmp/custom-services/metainfo.xml", ["\\{serviceVersion\\}": serviceVersion])
+    parsePackageVersion("common-services/SOLR/serviceVersion/service_advisor.py.template",
+            "${buildDir}/tmp/common-services/service_advisor.py", ["SOLRServiceAdvisor": 'SOLR' +
+            serviceVersion.replaceAll('\\.', '') + 'ServiceAdvisor'])
+}
+
+task makePackage(type: Tar, dependsOn: ["setPackageVersion"]) {
+    baseName = packageName
+    archiveName = packageName + '-mpack-' + serviceVersion + '.tar.gz'
+    compression = Compression.GZIP
+    extension = 'tar.gz'
+    destinationDir = file(buildDir)
+
+    into(packageName) {
+        into('') {
+            from buildDir.path + '/tmp/mpack.json'
+        }
+
+        into('common-services/SOLR/' + serviceVersion) {
+            from('common-services/SOLR/serviceVersion') {
+                include('**')
+                exclude('metainfo.xml.template')
+                exclude('service_advisor.py.template')
+            }
+            from buildDir.path + '/tmp/common-services/service_advisor.py',
+                    buildDir.path + '/tmp/common-services/metainfo.xml'
+        }
+
+        into('custom-services/SOLR/' + serviceVersion) {
+            from('custom-services/SOLR/serviceVersion') {
+                include('**')
+                exclude('metainfo.xml.template')
+            }
+            from buildDir.path + '/tmp/custom-services/metainfo.xml'
+        }
+    }
+}
diff --git a/common-services/SOLR/serviceVersion/configuration/example-collection.xml b/common-services/SOLR/serviceVersion/configuration/example-collection.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+    <property>
+        <name>solr_collection_sample_create</name>
+        <value>true</value>
+        <description>True to create a sample collection when Solr is deployed</description>
+        <display-name>Create sample collection</display-name>
+        <value-attributes>
+            <type>boolean</type>
+            <overridable>false</overridable>
+        </value-attributes>
+    </property>
+
+    <property>
+        <name>solr_collection_sample_name</name>
+        <value>collection1</value>
+        <description>Solr sample collection name. Mandatory</description>
+        <display-name>Sample collection name</display-name>
+    </property>
+
+    <property>
+        <name>solr_collection_sample_config_directory</name>
+        <value>data_driven_schema_configs</value>
+        <description>Solr sample collection configurations directory. Mandatory
+            This directory path is relative to /opt/lucidworks-hadoop/solr/server/solr/configset.
+            It must contain at least solrconfig.xml and schema.xml
+        </description>
+        <display-name>Solr configuration directory</display-name>
+    </property>
+
+    <property>
+        <name>solr_collection_sample_shards</name>
+        <value>2</value>
+        <description>Number of Solr shards, for details refer to
+            (https://cwiki.apache.org/confluence/display/solr/Shards+and+Indexing+Data+in+SolrCloud)
+        </description>
+        <value-attributes>
+            <type>int</type>
+        </value-attributes>
+        <display-name>Number of shards</display-name>
+    </property>
+
+    <property>
+        <name>solr_collection_sample_replicas</name>
+        <value>1</value>
+        <description>Number of Solr replicas, for details refer to
+            (https://cwiki.apache.org/confluence/display/solr/NRT%2C+Replication%2C+and+Disaster+Recovery+with+SolrCloud)
+        </description>
+        <value-attributes>
+            <type>int</type>
+        </value-attributes>
+        <display-name>Number of replicas</display-name>
+    </property>
+</configuration>
diff --git a/common-services/SOLR/serviceVersion/configuration/solr-cloud.xml b/common-services/SOLR/serviceVersion/configuration/solr-cloud.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+    <property>
+        <name>solr_cloud_enable</name>
+        <value>true</value>
+        <description>Whether Solr should be started in Cloud mode</description>
+        <display-name>Enable SolrCloud mode</display-name>
+        <value-attributes>
+            <type>boolean</type>
+            <overridable>false</overridable>
+        </value-attributes>
+    </property>
+
+    <property>
+        <name>solr_cloud_zk_directory</name>
+        <value>/solr</value>
+        <description>ZooKeeper directory (chroot) for shared Solr files</description>
+        <display-name>ZooKeeper root directory for Solr files</display-name>
+    </property>
+</configuration>