BIGTOP-4151: Add pkgSuffix and parentDir build arguments to Zookeeper…

… deb

bigtop-deploy/puppet/modules/hadoop_zookeeper/manifests/init.pp

@@ -17,6 +17,20 @@
   $kerberos_realm = "",
 ) {
 
+  $package_suffix = hiera("bigtop::package_suffix", "")
+  $parent_dir = hiera("bigtop::parent_dir", "")
+  $bigtop_base_version = hiera("bigtop::base_version", "")
+
+  if $package_suffix and $package_suffix != '' and $parent_dir and $parent_dir != '' {
+    $bin_home = "${parent_dir}/${bigtop_base_version}/usr/bin"
+  } else {
+    $bin_home = '/usr/bin'
+  }
+
+  $zookeeper_package = "zookeeper${package_suffix}"
+  $zookeeper_server_package = "zookeeper${package_suffix}-server"
+  $zookeeper_service_name = "zookeeper${package_suffix}-server"
+
   class deploy ($roles) {
     if ("zookeeper-client" in $roles) {
       include hadoop_zookeeper::client
@@ -45,15 +59,17 @@
   ) inherits hadoop_zookeeper {
     include hadoop_zookeeper::common
 
-    package { "zookeeper":
+
+
+    package { $zookeeper_package:
       ensure => latest,
       require => Package["jdk"],
     }
 
     if ($kerberos_realm and $kerberos_realm != "") {
       file { '/etc/zookeeper/conf/client-jaas.conf':
         content => template('hadoop_zookeeper/client-jaas.conf'),
-        require => Package['zookeeper'],
+        require => Package[$zookeeper_package],
       }
     }
   }
@@ -69,51 +85,60 @@
   ) inherits hadoop_zookeeper {
     include hadoop_zookeeper::common
 
-    package { "zookeeper-server":
+
+    package { $zookeeper_server_package:
       ensure => latest,
       require => Package["jdk"],
     }
 
-    service { "zookeeper-server":
+    file { $datadir:
+      ensure => directory,
+      owner  => 'zookeeper',
+      group  => 'zookeeper',
+      mode   => '0755',
+      require => Package[$zookeeper_server_package],
+    }
+
+    service { $zookeeper_service_name:
       ensure => running,
-      require => [ Package["zookeeper-server"],
+      require => [ Package[$zookeeper_server_package],
                    Exec["zookeeper-server-initialize"] ],
       subscribe => [ File["/etc/zookeeper/conf/zoo.cfg"],
-                     File["/var/lib/zookeeper/myid"] ],
+                     File["${datadir}/myid"] ],
       hasrestart => true,
       hasstatus => true,
     }
 
     file { "/etc/zookeeper/conf/zoo.cfg":
       content => template("hadoop_zookeeper/zoo.cfg"),
-      require => Package["zookeeper-server"],
+      require => Package[$zookeeper_server_package],
     }
 
-    file { "/var/lib/zookeeper/myid":
+    file { "${datadir}/myid":
       content => inline_template("<%= @myid %>"),
-      require => Package["zookeeper-server"],
+      require => [Package[$zookeeper_server_package], File[$datadir]],
     }
 
     exec { "zookeeper-server-initialize":
-      command => "/usr/bin/zookeeper-server-initialize",
+      command => "$bin_home/zookeeper-server-initialize",
       user    => "zookeeper",
-      creates => "/var/lib/zookeeper/version-2",
-      require => Package["zookeeper-server"],
+      creates => "${datadir}/version-2",
+      require => [Package[$zookeeper_server_package], File[$datadir]],
     }
 
     if ($kerberos_realm and $kerberos_realm != "") {
       require kerberos::client
 
       kerberos::host_keytab { "zookeeper":
         spnego  => true,
-        require => Package["zookeeper-server"],
-        before  => Service["zookeeper-server"],
+        require => Package[$zookeeper_server_package],
+        before  => Service[$zookeeper_service_name],
       }
 
       file { "/etc/zookeeper/conf/server-jaas.conf":
         content => template("hadoop_zookeeper/server-jaas.conf"),
-        require => Package["zookeeper-server"],
-        notify  => Service["zookeeper-server"],
+        require => Package[$zookeeper_server_package],
+        notify  => Service[$zookeeper_service_name],
       }
     }
   }

bigtop-packages/src/deb/zookeeper/control

@@ -20,27 +20,27 @@ Build-Depends: debhelper (>= 7.0.50~), autoconf, automake
 Standards-Version: 3.9.6
 Homepage: http://zookeeper.apache.org/
 
-Package: zookeeper
+Package: zookeeper-pkgsuffix
 Architecture: all
 Depends: adduser, bigtop-utils (>= 0.7)
 Description: A high-performance coordination service for distributed applications.
  ZooKeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services.  All of these kinds of services are used in some form or another by distributed applications. Each time they are implemented there is a lot of work that goes into fixing the bugs and race conditions that are inevitable. Because of the difficulty of implementing these kinds of services, applications initially usually skimp on them ,which make them brittle in the presence of change and difficult to manage. Even when done correctly, different implementations of these services lead to management complexity when the applications are deployed.
 
-Package: zookeeper-server
+Package: zookeeper-pkgsuffix-server
 Architecture: all
-Depends: zookeeper (= ${source:Version})
+Depends: zookeeper-pkgsuffix (= ${source:Version})
 Description: The Hadoop Zookeeper server
  This package starts the zookeeper server on startup
 
-Package: zookeeper-rest
+Package: zookeeper-pkgsuffix-rest
 Architecture: all
-Depends: zookeeper (= ${source:Version}), lsb-base
+Depends: zookeeper-pkgsuffix (= ${source:Version}), lsb-base
 Description: The Zookeeper REST server
  This package starts the zookeeper REST server on startup
 
-Package: zookeeper-native
+Package: zookeeper-pkgsuffix-native
 Architecture: any
-Depends: zookeeper (= ${source:Version}),  ${shlibs:Depends} 
+Depends: zookeeper-pkgsuffix (= ${source:Version}),  ${shlibs:Depends}
 Description: C bindings for ZooKeeper clients
  Provides native libraries and development headers for C / C++ ZooKeeper clients. Consists of both single-threaded and multi-threaded implementations.
 

bigtop-packages/src/deb/zookeeper/rules

@@ -28,24 +28,77 @@ zookeeper_pkg_name=zookeeper
 %:
 	dh $@
 
+parent_dir=${PARENT_DIR}
+pkg_name_suffix=${PKG_NAME_SUFFIX}
+
+zookeeper_name=zookeeper
+doc_zookeeper=${parent_dir}/usr/share/doc/${zookeeper_name}
+usr_lib_zookeeper=${parent_dir}/usr/lib/${zookeeper_name}
+var_lib_zookeeper=${parent_dir}/var/lib/${zookeeper_name}
+bin_dir=${parent_dir}/usr/bin
+man_dir=${parent_dir}/usr/share/man
+etc_zookeeper=${parent_dir}/etc/${zookeeper_name}
+etc_zookeeper_conf_dist=${parent_dir}/etc/${zookeeper_name}/conf.dist
+etc_default=${parent_dir}/etc/default
+etc_default_zookeeper=${etc_default}/${zookeeper_name}
+include_dir=${parent_dir}/usr/include
+lib_dir=${parent_dir}/usr/lib/
+
+define gen_rule
+    $(foreach item,postinst preinst prerm,\
+        if [ -f debian/zookeeper$(1).$(item) ]; then \
+            sed -i -e 's:@etc_zookeeper@:${etc_zookeeper}:g' -e 's:@var_lib_zookeeper@:${var_lib_zookeeper}:g' debian/zookeeper$(1).$(item); \
+            if [ "${pkg_name_suffix}" != "" ] ;then mv debian/zookeeper$(1).$(item) debian/zookeeper${pkg_name_suffix}$(1).$(item);fi \
+        fi; \
+    )
+endef
+
+.PHONY: update_control
+.PHONY: gen_files
+
+update_control:
+	sed -i 's/-pkgsuffix/${pkg_name_suffix}/g' debian/control
+	if [ "${pkg_name_suffix}" != "" ] && [ -e debian/zookeeper-server.init ]; then mv debian/zookeeper-server.init debian/zookeeper${pkg_name_suffix}-server.init; fi
+	sed -i  -e 's:@etc_default_zookeeper:${etc_default_zookeeper}:g' debian/zookeeper${pkg_name_suffix}-server.init
+	sed -i -e 's:@bin_home:${bin_dir}:g' debian/zookeeper${pkg_name_suffix}-server.init
+
+
+gen_files:
+	$(call gen_rule,)
+
+
+override_dh_auto_clean: update_control
+	dh_auto_clean
+
 override_dh_auto_build:
 	bash debian/do-component-build -Divy.home=${HOME}/.ivy2
 	mkdir -p debian/tmp
 	tar cf - --exclude=debian/\* . | (cd debian/tmp && tar xf -)
 
-override_dh_auto_install:
+override_dh_auto_install: gen_files
 	cp debian/zookeeper.1 debian/zoo.cfg debian/zookeeper.default .
 	bash -x debian/install_zookeeper.sh \
-	  --build-dir=build \
-	  --prefix=debian/$(zookeeper_pkg_name)
+		--build-dir=build \
+		--prefix=debian/$(zookeeper_pkg_name)${pkg_name_suffix} \
+		--doc-dir=${doc_zookeeper} \
+		--lib-dir=${usr_lib_zookeeper} \
+		--bin-dir=${bin_dir} \
+		--man-dir=${man_dir} \
+		--conf-dist-dir=${etc_zookeeper_conf_dist} \
+		--etc-default=${etc_default} \
+		--system-include-dir=${include_dir} \
+		--system-lib-dir=${lib_dir}
 	   # Move native files to a dedicated package
-	   mkdir -p debian/zookeeper-native/usr/bin
-	   mkdir -p debian/zookeeper-native/usr/lib
-	   mv debian/zookeeper/usr/bin/cli_* debian/zookeeper/usr/bin/load_gen debian/zookeeper-native/usr/bin/
-	   mv debian/zookeeper/usr/lib/libzookeeper* debian/zookeeper-native/usr/lib/
-	   mv debian/zookeeper/usr/lib/zookeeper-native debian/zookeeper-native/usr/lib/
-	   mv debian/zookeeper/usr/include debian/zookeeper-native/usr/include
+	   mkdir -p debian/zookeeper${pkg_name_suffix}-native/${bin_dir}
+	   mkdir -p debian/zookeeper${pkg_name_suffix}-native/${lib_dir}
+	   mv debian/zookeeper${pkg_name_suffix}/${bin_dir}/cli_* debian/zookeeper${pkg_name_suffix}/${bin_dir}/load_gen debian/zookeeper${pkg_name_suffix}-native/${bin_dir}
+	   mv debian/zookeeper${pkg_name_suffix}/${lib_dir}/libzookeeper* debian/zookeeper${pkg_name_suffix}-native/${lib_dir}
+	   mv debian/zookeeper${pkg_name_suffix}/${lib_dir}/zookeeper-native debian/zookeeper${pkg_name_suffix}-native/${lib_dir}
+	   mv debian/zookeeper${pkg_name_suffix}/${include_dir} debian/zookeeper${pkg_name_suffix}-native/${include_dir}
 	bash debian/init.d.tmpl debian/zookeeper-rest.svc deb debian/zookeeper-rest.init
 
 override_dh_strip:
 	dh_strip --no-automatic-dbgsym
+
+
+override_dh_strip_nondeterminism:

bigtop-packages/src/deb/zookeeper/zookeeper-server.init

@@ -34,10 +34,10 @@ set -e
 # Autodetect JAVA_HOME if not defined
 . /usr/lib/bigtop-utils/bigtop-detect-javahome
 
-. /etc/default/zookeeper
+. @etc_default_zookeeper
 
 PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-DAEMON_SCRIPT="/usr/bin/zookeeper-server"
+DAEMON_SCRIPT="@bin_home/zookeeper-server"
 
 NAME=zookeeper-server
 DESC="ZooKeeper daemon"

bigtop-packages/src/deb/zookeeper/zookeeper.postinst

@@ -25,7 +25,7 @@ case "$1" in
     configure)
         # Install config alternatives
         chown -R zookeeper:zookeeper /var/log/zookeeper
-        update-alternatives  --install /etc/${zookeeper_pkg_name}/conf ${zookeeper_pkg_name}-conf /etc/${zookeeper_pkg_name}/conf.dist 30
+        update-alternatives  --install /etc/${zookeeper_pkg_name}/conf ${zookeeper_pkg_name}-conf @etc_zookeeper@/conf.dist 30
     ;;
 
     abort-upgrade|abort-remove|abort-deconfigure)

bigtop-packages/src/deb/zookeeper/zookeeper.preinst

@@ -38,13 +38,13 @@ case "$1" in
                 adduser \
                   --system \
                   --ingroup zookeeper \
-                  --home /var/lib/zookeeper \
+                  --home @var_lib_zookeeper@ \
                   --gecos "ZooKeeper User" \
                   --shell /bin/false \
                   zookeeper >/dev/null
         fi
         install -d -m 0755 -o zookeeper -g zookeeper /var/log/zookeeper
-        install -d -m 0755 -o zookeeper -g zookeeper /var/lib/zookeeper
+        install -d -m 0755 -o zookeeper -g zookeeper @var_lib_zookeeper@
     ;;
 
     abort-upgrade)

bigtop-packages/src/deb/zookeeper/zookeeper.prerm

@@ -25,7 +25,7 @@ set -e
 
 case "$1" in
     remove|upgrade|deconfigure)
-      update-alternatives --remove ${zookeeper_pkg_name}-conf /etc/${zookeeper_pkg_name}/conf.dist || :
+      update-alternatives --remove ${zookeeper_pkg_name}-conf @etc_zookeeper@/conf.dist || :
     ;;
 
     failed-upgrade)

bigtop-tests/smoke-tests/zookeeper/TestZookeeper.groovy

@@ -29,7 +29,7 @@ import static org.apache.bigtop.itest.LogErrorsUtils.logError
 
 class TestZookeeper {
   static private Log LOG = LogFactory.getLog(Object.class)
-
+  static final String ZK_HOME = System.getenv("ZOOKEEPER_HOME") ?: "/usr/lib/zookeeper"
   static Shell sh = new Shell("/bin/bash -s")
 
   @BeforeClass
@@ -47,7 +47,7 @@ class TestZookeeper {
     // Basic test to verify that the server is running, and is in a
     // state that we expect.
     LOG.info('Running zkServer.sh status');
-    sh.exec("/usr/lib/zookeeper/bin/zkServer.sh status");
+    sh.exec("${ZK_HOME}/bin/zkServer.sh status");
     logError(sh);
     assertTrue("Failed ...", sh.getRet() == 0);
 

bigtop.bom

@@ -136,6 +136,7 @@ bigtop {
       name    = 'zookeeper'
       pkg     = name
       rpm_pkg_suffix = "_" + bigtop.base_version.replace(".", "_")
+      deb_pkg_suffix = "-" + bigtop.base_version.replace(".", "-")
       version {
         base  = '3.7.2'
         pkg   = base

packages.gradle

@@ -382,6 +382,9 @@ def genTasks = { target ->
     def final BASE_VERSION = config.bigtop.components[target].version.base
     def final SRCDEB = "${PKG_NAME}_$PKG_VERSION-${BIGTOP_BUILD_STAMP}.dsc"
     def final HADOOP_VERSION = config.bigtop.components["hadoop"].version.pkg
+    def final FULL_PARENT_DIR = getParentDir(config.bigtop.base_version, "deb")
+    def final DEB_PKG_NAME_SUFFIX =  getPkgNameSuffix(config.bigtop.components[target], "deb").pkgNameSuffix
+    def final BIGTOP_BASE_VERSION = "${config.bigtop.base_version}"
 
     exec {
       workingDir PKG_OUTPUT_DIR
@@ -399,6 +402,9 @@ def genTasks = { target ->
 --set-envvar=${toOldStyleName(target)}_BASE_VERSION=$BASE_VERSION \
 --set-envvar=${toOldStyleName(target)}_VERSION=$PKG_VERSION \
 --set-envvar=${toOldStyleName(target)}_RELEASE=$BIGTOP_BUILD_STAMP \
+--set-envvar=PARENT_DIR=$FULL_PARENT_DIR \
+--set-envvar=PKG_NAME_SUFFIX=$DEB_PKG_NAME_SUFFIX \
+--set-envvar=bigtop_base_version=${BIGTOP_BASE_VERSION} \
 -uc -us -b
 """
     exec {
@@ -428,7 +434,8 @@ def genTasks = { target ->
       println "\tNothing to do. Exiting..."
       return
     }
-     def final BIGTOP_BUILD_STAMP = System.getenv('BIGTOP_BUILD_STAMP') ?:
+    def final BIGTOP_BASE_VERSION = "${config.bigtop.base_version}"
+    def final BIGTOP_BUILD_STAMP = System.getenv('BIGTOP_BUILD_STAMP') ?:
              config.bigtop.components[target].version.release
     def final TARBALL_SRC = config.bigtop.components[target].tarball.source
     def final TARBALL_DST = config.bigtop.components[target].tarball.destination ?: TARBALL_SRC
@@ -443,6 +450,8 @@ def genTasks = { target ->
     def final DEB_PKG_DIR = "$PKG_BUILD_DIR/deb/$PKG_NAME-${PKG_VERSION}-${BIGTOP_BUILD_STAMP}"
     def final ENABLE_MAVEN_PARALLEL_BUILD = config.bigtop.components[target].maven_parallel_build
     def final MAVEN_BUILD_THREADS = project.hasProperty('buildThreads') ? project.property('buildThreads') : null
+    def final FULL_PARENT_DIR = getParentDir(config.bigtop.base_version, "deb")
+    def final DEB_PKG_NAME_SUFFIX =  getPkgNameSuffix(config.bigtop.components[target], "deb").pkgNameSuffix
 
     mkdir (DEB_BLD_DIR)
     copy {
@@ -511,6 +520,9 @@ def genTasks = { target ->
     // Creating source package
     exec {
       workingDir DEB_BLD_DIR
+      environment 'PARENT_DIR', FULL_PARENT_DIR
+      environment 'PKG_NAME_SUFFIX', DEB_PKG_NAME_SUFFIX
+      environment 'bigtop_base_version', BIGTOP_BASE_VERSION
       commandLine "dpkg-buildpackage -uc -us -sa -S".split(' ')
     }
     mkdir(PKG_OUTPUT_DIR)