Updated to support installing HDF 2.0.0.0 (Nifi 1.0) on Ambari 2.4

README.md

@@ -3,16 +3,21 @@ Ambari service for easily installing and managing NiFi on HDP cluster and viewin
 
 Features:
 
-- By default, downloads the current GA version - HDF 1.2.0.1 package (nifi 0.6.0) - but also gives option to build the latest Nifi from source instead
+- By default, downloads the current GA version 
 - Exposes nifi.properties, bootstrap.conf, logback.xml in Ambari UI (so you can configure port, memory, log dir etc)
 - Sets up initial flow.xml.gz that sets up Ambari reporting task to send Ambari metrics
 - Includes metrics widgets from [here](https://cwiki.apache.org/confluence/display/NIFI/Ambari+Metrics)
 
 Updates:
-- Jul-10-2016: updated to support installing on Ambari 2.4/HDP 2.5 tech preview build/VM
+- Sept-14-2016: updated to support installing HDF 2.0.0.0 (Nifi 1.0) on Ambari 2.4/HDP 2.5 tech preview build/VM
+- Jul-10-2016: updated to support installing HDF 1.2.0.0 on Ambari 2.4/HDP 2.5 tech preview build/VM
+
+Installing on older Ambari versions:
+- Refer to [archive folder](https://github.com/abajwa-hw/ambari-nifi-service/tree/master/configuration) for older versions of the service definition 
+  - To install on Ambari 2.4 and earlier use HDF-1.2.0.0-Nifi-0.6.0.1.2.0.1.zip
 
-Limitations:
 
+Limitations:
 - This is not an officially supported service and *is not meant to be deployed in production systems*. It is only meant for testing demo/purposes
 - It does not support Ambari/HDP upgrade process and will cause upgrade problems if not removed prior to upgrade
 - Not tested on secured clusters
@@ -25,8 +30,8 @@ Authors:
 
 #### Option 1: Deploy Nifi on existing cluster
 
-- Download HDP 2.4 sandbox VM image (Hortonworks_sanbox_with_hdp_2_4_vmware.ova) from [Hortonworks website](http://hortonworks.com/products/hortonworks-sandbox/)
-- Import Hortonworks_sanbox_with_hdp_2_4_vmware.ova into VMWare and set the VM memory size to 8GB
+- Download HDP 2.5 sandbox VM image  from [Hortonworks website](http://hortonworks.com/products/hortonworks-sandbox/)
+- Import ova file into VMWare and set the VM memory size to 8GB
 - Now start the VM
 - After it boots up, find the IP address of the VM and add an entry into your machines hosts file. For example:
 ```

configuration/nifi-ambari-config.xml

@@ -10,13 +10,19 @@
     <value>/opt</value>
     <description>Local dir where to install component. NiFi folder will be created as a subdir of this dir e.g. /opt/HDF-x.x.x.x</description>
   </property> 
-  
+
   <property>
     <name>nifi.port</name>
     <value>9090</value>
     <description>Port NiFi runs on</description>
   </property>
 
+  <property>
+    <name>nifi.node.protocol.port</name>
+    <value>9088</value>
+    <description>HTTP port NiFi node protocol runs on</description>
+  </property>
+
   <property>
     <name>nifi.initial_mem</name>
     <value>512m</value>
@@ -45,4 +51,4 @@
 
 
 
-</configuration>  
+</configuration>  

configuration/nifi-bootstrap-env.xml

@@ -105,7 +105,10 @@ notification.max.attempts=5
 
   </value>
   <description>Template for bootstrap.conf</description>
-
+  <value-attributes>
+    <type>content</type>
+  </value-attributes>
+  <on-ambari-upgrade add="true"/>  
   </property>
 
 </configuration>

configuration/nifi-env.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+
+  <property>
+    <name>content</name>
+    <display-name>Template for nifi-env.sh</display-name>         
+    <value>
+#!/bin/sh
+
+# The java implementation to use.
+export JAVA_HOME={{jdk64_home}}
+
+export NIFI_HOME={{nifi_dir}}
+
+#The directory for the NiFi pid file
+export NIFI_PID_DIR="{{nifi_pid_dir}}"
+
+#The directory for NiFi log files
+export NIFI_LOG_DIR="{{nifi_log_dir}}"
+
+
+  </value>
+  <description>Template for nifi-env.sh</description>
+  <value-attributes>
+    <type>content</type>
+  </value-attributes>  
+  <on-ambari-upgrade add="true"/>
+  </property>
+
+</configuration>
+

configuration/nifi-flow-env.xml

@@ -42,7 +42,10 @@
 </flowController>
   </value>
   <description>Initial template for flow.xml. Only used during initial install of Nifi service. Post-install this can be reconfigured from Nifi UI under Controller Settings</description>
-
+  <value-attributes>
+    <type>content</type>
+  </value-attributes>
+  <on-ambari-upgrade add="true"/>  
   </property>
 
 </configuration>

configuration/nifi-logback-env.xml → configuration/nifi-logback-env.xml.bak

@@ -150,7 +150,10 @@
 
   </value>
   <description>Initial template for flow.xml. Only used during initial install of Nifi service. Post-install this can be reconfigured from Nifi UI under Controller Settings</description>
-
+  <value-attributes>
+    <type>content</type>
+  </value-attributes>
+  <on-ambari-upgrade add="true"/>  
   </property>
 
 </configuration>

configuration/nifi-properties-env.xml

@@ -8,17 +8,20 @@
     <description>This is the jinja template for nifi.properties file</description>
     <value>      
 # Core Properties #
-nifi.version=0.6.0.1.2.0.1-1
+nifi.version={{nifi_dirname}}
 nifi.flow.configuration.file=./conf/flow.xml.gz
+nifi.flow.configuration.archive.enabled=true
 nifi.flow.configuration.archive.dir=./conf/archive/
+nifi.flow.configuration.archive.max.time=30 days
+nifi.flow.configuration.archive.max.storage=500 MB
 nifi.flowcontroller.autoResumeState=true
 nifi.flowcontroller.graceful.shutdown.period=10 sec
 nifi.flowservice.writedelay.interval=500 ms
 nifi.administrative.yield.duration=30 sec
 # If a component has no work to do (is "bored"), how long should we wait before checking again for work?
 nifi.bored.yield.duration=10 millis
 
-nifi.authority.provider.configuration.file=./conf/authority-providers.xml
+nifi.authority.provider.configuration.file=./conf/authorizers.xml
 nifi.login.identity.provider.configuration.file=./conf/login-identity-providers.xml
 nifi.templates.directory=./conf/templates
 nifi.ui.banner.text=
@@ -105,9 +108,11 @@ nifi.components.status.repository.buffer.size=1440
 nifi.components.status.snapshot.frequency=1 min
 
 # Site to Site properties
-nifi.remote.input.socket.host=
+nifi.remote.input.host=
+nifi.remote.input.secure=false
 nifi.remote.input.socket.port=
-nifi.remote.input.secure=true
+nifi.remote.input.http.enabled=true
+nifi.remote.input.http.transaction.ttl=30 sec
 
 # web properties #
 nifi.web.war.directory=./lib
@@ -120,8 +125,10 @@ nifi.web.jetty.threads=200
 
 # security properties #
 nifi.sensitive.props.key=
+nifi.sensitive.props.key.protected=
 nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
 nifi.sensitive.props.provider=BC
+nifi.sensitive.props.additional.keys=
 
 nifi.security.keystore=
 nifi.security.keystoreType=
@@ -131,56 +138,63 @@ nifi.security.truststore=
 nifi.security.truststoreType=
 nifi.security.truststorePasswd=
 nifi.security.needClientAuth=
-nifi.security.user.credential.cache.duration=24 hours
-nifi.security.user.authority.provider=file-provider
+nifi.security.user.authorizer=file-provider
 nifi.security.user.login.identity.provider=
-nifi.security.support.new.account.requests=
-# Valid Authorities include: ROLE_MONITOR,ROLE_DFM,ROLE_ADMIN,ROLE_PROVENANCE,ROLE_NIFI
-nifi.security.anonymous.authorities=
 nifi.security.ocsp.responder.url=
 nifi.security.ocsp.responder.certificate=
 
-# cluster common properties (cluster manager and nodes must have same values) #
+# Identity Mapping Properties #
+# These properties allow normalizing user identities such that identities coming from different identity providers
+# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing
+# DNs from certificates and principals from Kerberos into a common identity string:
+#
+# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$
+# nifi.security.identity.mapping.value.dn=$1@$2
+# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$
+# nifi.security.identity.mapping.value.kerb=$1@$2
+
+# cluster common properties (all nodes must have same values) #
 nifi.cluster.protocol.heartbeat.interval=5 sec
 nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
 
 # cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
+nifi.cluster.is.node=true
+nifi.cluster.node.address={{nifi_node_host}}
+nifi.cluster.node.protocol.port={{nifi_node_protocol_port}}
+nifi.cluster.node.protocol.threads=10
+nifi.cluster.node.event.history.size=25
+nifi.cluster.node.connection.timeout=5 sec
+nifi.cluster.node.read.timeout=5 sec
+nifi.cluster.firewall.file=
+nifi.cluster.flow.election.max.wait.time=5 mins
+nifi.cluster.flow.election.max.candidates={{nifi_num_nodes}}
+
+# zookeeper properties, used for cluster management #
+nifi.zookeeper.connect.string={{zookeeper_quorum}}
+nifi.zookeeper.connect.timeout=3 secs
+nifi.zookeeper.session.timeout=3 secs
+nifi.zookeeper.root.node=/nifi
 
 # kerberos #
 nifi.kerberos.krb5.file=
+
+# kerberos service principle #
 nifi.kerberos.service.principal=
-nifi.kerberos.keytab.location=
-nifi.kerberos.authentication.expiration=12 hours
+nifi.kerberos.service.keytab.location=
+
+# kerberos spnego principle #
+nifi.kerberos.spnego.principal=
+nifi.kerberos.spnego.keytab.location=
+nifi.kerberos.spnego.authentication.expiration=12 hours
+
+# external properties files for variable registry
+# supports a comma delimited list of file locations
+nifi.variable.registry.properties=
   </value>
+  <value-attributes>
+    <type>content</type>
+  </value-attributes>
+  <on-ambari-upgrade add="true"/>
   </property>  
 
 </configuration>