Merge pull request #1647 from apache/4.9-systemdubuntupkging

[lts] CLOUDSTACK-9462: Systemd support for Ubuntu 16.04Created this based on @wido 's origin PR #1541 .

Requesting for review and testing -- @jburwell @karuturi @wido @vincentbernat
@wido I think this change only brings systemd support to agent and usage packages, or does cloudstack-management pkg has systemd support too?

@blueorangutan package

- systemd: Add a /etc/sysconfig/cloudstack-* file

  This allows users to easily override variables passed to Java when
  starting up.

  It also creates a foundation for sharing the systemd service profile
  between CentOS and Ubuntu since it only requires the environment file
  to be changed.

- deb: Add Ubuntu 16.04 support

  Ubuntu 16.04 differs from Ubuntu 14.04 in a few ways:
  - systemd instead of sysvinit / upstart
  - Java 8 support

  The packaging now detects on which distribution it is being
  build and based on that it installs different files in the
  packages, but it also changes the Dependencies.

* pr/1647:
  CLOUDSTACK-9462: Refactor systemd scripts
  CLOUDSTACK-9462: Systemd support for Ubuntu 16.04

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

Author: yadvr

debian/cloudstack-agent.dirs

@@ -0,0 +1 @@
+/var/log/cloudstack/agent

debian/cloudstack-agent.install

@@ -18,12 +18,12 @@
 /etc/cloudstack/agent/agent.properties
 /etc/cloudstack/agent/environment.properties
 /etc/cloudstack/agent/log4j-cloud.xml
+/etc/default/cloudstack-agent
 /etc/profile.d/cloudstack-agent-profile.sh
 /etc/logrotate.d/cloudstack-agent
 /etc/init.d/cloudstack-agent
 /usr/bin/cloudstack-setup-agent
 /usr/bin/cloudstack-ssh
 /usr/bin/cloudstack-agent-upgrade
-/var/log/cloudstack/agent
 /usr/share/cloudstack-agent/lib/*
 /usr/share/cloudstack-agent/plugins

debian/cloudstack-usage.dirs

@@ -0,0 +1 @@
+/var/log/cloudstack/usage

debian/cloudstack-usage.install

@@ -18,5 +18,5 @@
 /usr/share/cloudstack-usage/lib/*
 /usr/share/cloudstack-usage/plugins
 /etc/init.d/cloudstack-usage
-/var/log/cloudstack/usage
 /etc/cloudstack/usage/*
+/etc/default/cloudstack-usage

debian/control

@@ -2,8 +2,8 @@ Source: cloudstack
 Section: libs
 Priority: extra
 Maintainer: Wido den Hollander <wido@widodh.nl>
-Build-Depends: debhelper (>= 9), openjdk-8-jdk | openjdk-7-jdk, genisoimage,
- python-mysql.connector, maven (>= 3) | maven3, python (>= 2.7)
+Build-Depends: debhelper (>= 9), openjdk-8-jdk | java8-sdk | java8-jdk | openjdk-7-jdk, genisoimage,
+ python-mysql.connector, maven (>= 3) | maven3, python (>= 2.7), lsb-release, dh-systemd
 Standards-Version: 3.8.1
 Homepage: http://www.cloudstack.org/
 
@@ -22,7 +22,8 @@ Description: CloudStack server library
 
 Package: cloudstack-agent
 Architecture: all
-Depends: ${misc:Depends}, ${python:Depends}, openjdk-8-jre-headless | openjdk-7-jre-headless, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 0.9.8), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables, lsb-release
+Depends: ${python:Depends}, openjdk-8-jre-headless | java8-runtime-headless | java8-runtime | openjdk-7-jre-headless, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 0.9.8), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables, lsb-release
+Recommends: init-system-helpers
 Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
 Description: CloudStack agent
  The CloudStack agent is in charge of managing shared computing resources in
@@ -31,7 +32,8 @@ Description: CloudStack agent
 
 Package: cloudstack-usage
 Architecture: all
-Depends: ${misc:Depends}, openjdk-8-jre-headless | openjdk-7-jre-headless, cloudstack-common (= ${source:Version}), jsvc, libmysql-java
+Depends: openjdk-8-jre-headless | java8-runtime-headless | java8-runtime | openjdk-7-jre-headless, cloudstack-common (= ${source:Version}), jsvc, libmysql-java
+Recommends: init-system-helpers
 Description: CloudStack usage monitor
  The CloudStack usage monitor provides usage accounting across the entire cloud for
  cloud operators to charge based on usage parameters.

debian/rules

@@ -6,7 +6,7 @@ SYSCONFDIR = "/etc"
 DESTDIR = "debian/tmp"
 
 %:
-	dh $@ --with python2
+	dh $@ --with python2,systemd
 
 override_dh_auto_configure:
 	cp packaging/debian/replace.properties replace.properties.tmp
@@ -26,6 +26,8 @@ override_dh_auto_install:
 	# Common packages
 	mkdir -p $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)
 	mkdir -p $(DESTDIR)/$(SYSCONFDIR)/init.d
+	mkdir -p $(DESTDIR)/$(SYSCONFDIR)/default
+
 	mkdir -p $(DESTDIR)/var/cache/$(PACKAGE)
 	mkdir -p $(DESTDIR)/var/log/$(PACKAGE)
 	mkdir -p $(DESTDIR)/var/lib/$(PACKAGE)
@@ -35,13 +37,20 @@ override_dh_auto_install:
 	# cloudstack-agent
 	mkdir $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/agent
 	mkdir $(DESTDIR)/$(SYSCONFDIR)/profile.d
-	mkdir $(DESTDIR)/var/log/$(PACKAGE)/agent
 	mkdir $(DESTDIR)/usr/share/$(PACKAGE)-agent
 	mkdir $(DESTDIR)/usr/share/$(PACKAGE)-agent/plugins
 	install -D agent/target/cloud-agent-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/$(PACKAGE)-agent.jar
 	install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/
 	install -D plugins/hypervisors/kvm/target/dependencies/* $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/
-	install -D packaging/debian/init/cloud-agent $(DESTDIR)/$(SYSCONFDIR)/init.d/$(PACKAGE)-agent
+
+	install -m0755 packaging/debian/$(PACKAGE)-agent.init $(DESTDIR)/$(SYSCONFDIR)/init.d/$(PACKAGE)-agent
+	install -d -m0755 debian/$(PACKAGE)-agent/lib/systemd/system
+	# Fix libvirt service name for Debian/Ubuntu
+	sed -i 's/Requires=libvirtd.service/Requires=libvirt-bin.service/g' packaging/systemd/$(PACKAGE)-agent.service
+	sed -i 's/After=libvirtd.service/After=libvirt-bin.service/g' packaging/systemd/$(PACKAGE)-agent.service
+	install -m0644 packaging/systemd/$(PACKAGE)-agent.service debian/$(PACKAGE)-agent/lib/systemd/system/$(PACKAGE)-agent.service
+	install -m0644 packaging/systemd/$(PACKAGE)-agent.default $(DESTDIR)/$(SYSCONFDIR)/default/$(PACKAGE)-agent
+
 	install -D agent/target/transformed/cloud-setup-agent $(DESTDIR)/usr/bin/cloudstack-setup-agent
 	install -D agent/target/transformed/cloud-ssh $(DESTDIR)/usr/bin/cloudstack-ssh
 	install -D agent/target/transformed/cloudstack-agent-profile.sh $(DESTDIR)/$(SYSCONFDIR)/profile.d/cloudstack-agent-profile.sh
@@ -116,14 +125,17 @@ override_dh_auto_install:
 
 	# cloudstack-usage
 	mkdir $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/usage
-	mkdir $(DESTDIR)/var/log/$(PACKAGE)/usage
 	mkdir $(DESTDIR)/usr/share/$(PACKAGE)-usage
 	mkdir $(DESTDIR)/usr/share/$(PACKAGE)-usage/plugins
 	install -D usage/target/cloud-usage-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-usage/lib/$(PACKAGE)-usage.jar
 	install -D usage/target/dependencies/* $(DESTDIR)/usr/share/$(PACKAGE)-usage/lib/
 	cp usage/target/transformed/db.properties $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/usage/
 	cp usage/target/transformed/log4j-cloud_usage.xml $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/usage/log4j-cloud.xml
-	install -D packaging/debian/init/cloud-usage $(DESTDIR)/$(SYSCONFDIR)/init.d/$(PACKAGE)-usage
+
+	install -D -m0755 packaging/debian/$(PACKAGE)-usage.init $(DESTDIR)/$(SYSCONFDIR)/init.d/$(PACKAGE)-usage
+	install -d -m0755 debian/$(PACKAGE)-usage/lib/systemd/system
+	install -m0644 packaging/systemd/$(PACKAGE)-usage.service debian/$(PACKAGE)-usage/lib/systemd/system/$(PACKAGE)-usage.service
+	install -m0644 packaging/systemd/$(PACKAGE)-usage.default $(DESTDIR)/$(SYSCONFDIR)/default/$(PACKAGE)-usage
 
 	# cloudstack-marvin
 	mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-marvin
@@ -136,5 +148,8 @@ override_dh_auto_install:
 override_dh_installinit:
 	dh_installinit -pcloudstack-management -pcloudstack-agent -pcloudstack-usage --onlyscripts --no-start
 
+override_dh_systemd_enable:
+	dh_systemd_enable -pcloudstack-agent -pcloudstack-usage
+
 override_dh_installdocs:
 	dh_installdocs -A tools/whisker/LICENSE tools/whisker/NOTICE INSTALL.md

packaging/README.md

@@ -0,0 +1,39 @@
+# CloudStack RPM and DEB packaging
+This directory contains all the required scripts and tools needed to build RPM and DEB packages for Apache CloudStack.
+
+These scripts are also used by the CloudStack team to build packages for the official release of CloudStack.
+
+# Requirements
+The RPM and DEB packages have dependencies on versions of specific libraries. Due to these dependencies the following distributions and their versions are supported by the packages.
+
+* CentOS / RHEL: 6 and 7
+* Debian 7 (Wheezy) and 8 (Jessy) (untested!)
+* Ubuntu: 14.04 (Trusty) and 16.04 (Xenial)
+
+# Building
+Using the scripts in the *packaging* directory the RPM and DEB packages can be build.
+
+## DEB
+If you simply want to build packages go to the root directory of your CloudStack source code and run:
+
+``dpkg-buildpackage``
+
+This will build packages for the current distribution version you are running. If you run this on a Ubuntu 16.04 system the packages will be tailored for Ubuntu 16.04 and will not install on Ubuntu 14.04.
+
+### Building cross-distribution
+If you want to build packages for a different distribution run the *build-deb.sh* script. This will build packages with the current distribution as a suffix to the package names. E.g. *cloudstack-agent_4.9.0~xenial_all.deb*
+
+Using a Docker image you can build packages for a distribution you are not running.
+
+The following commands assume that the CloudStack source is present in **/tmp/cloudstack** on the system you are running these commands on.
+
+``docker run -ti -v /tmp:/src ubuntu:16.04 /bin/bash -c "apt-get update && apt-get install -y dpkg-dev python debhelper openjdk-8-jdk genisoimage python-mysql.connector maven lsb-release devscripts && /src/cloudstack/packaging/build-deb.sh"``
+
+``docker run -ti -v /tmp:/src ubuntu:14.04 /bin/bash -c "apt-get update && apt-get install -y dpkg-dev python debhelper openjdk-7-jdk genisoimage python-mysql.connector maven lsb-release devscripts && /src/cloudstack/packaging/build-deb.sh"``
+
+The commands above will generate Ubuntu 14.04 and 16.04 packages which you will find in */tmp* on your system after the build succeeds.
+
+## RPM
+The *package.sh* script can be used to build RPM packages for CloudStack. In the *packaging* script you can run the following command:
+
+``./package.sh --pack oss --distribution centos7``

packaging/build-deb.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+#
+# This script builds Debian packages for CloudStack and does
+# so by altering the debian/changelog file and add the Ubuntu
+# version as a suffix, for example: 4.9.0~xenial
+#
+# To build packages for Ubuntu 14.04 run this script on a
+# 14.04 system. The same goes for Ubuntu 16.04
+#
+# The biggest difference between those two versions is the
+# sysvinit vs systemd and Java 7 vs Java 8
+#
+# These packages can be build using Docker for example:
+#
+# Assume that the cloudstack source is present in /tmp/cloudstack
+#
+# Ubuntu 16.04
+# docker run -ti -v /tmp:/src ubuntu:16.04 /bin/bash -c "apt-get update && apt-get install -y dpkg-dev python debhelper openjdk-8-jdk genisoimage python-mysql.connector maven lsb-release devscripts && /src/cloudstack/packaging/build-deb.sh"
+#
+# Ubuntu 14.04
+# docker run -ti -v /tmp:/src ubuntu:14.04 /bin/bash -c "apt-get update && apt-get install -y dpkg-dev python debhelper openjdk-7-jdk genisoimage python-mysql.connector maven lsb-release devscripts && /src/cloudstack/packaging/build-deb.sh"
+#
+
+cd `dirname $0`
+cd ..
+
+dpkg-checkbuilddeps
+
+VERSION=$(grep '^  <version>' pom.xml| cut -d'>' -f2 |cut -d'<' -f1)
+DISTCODE=$(lsb_release -sc)
+
+dch -b -v "${VERSION}~${DISTCODE}" -u low -m "Apache CloudStack Release ${VERSION}"
+
+dpkg-buildpackage -j2 -b -uc -us

packaging/centos7/cloud.spec

@@ -210,7 +210,7 @@ mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/temp
 mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/mnt
 mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/management
 mkdir -p ${RPM_BUILD_ROOT}%{_initrddir}
-mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/sysconfig
+mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/default
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/profile.d
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d
 
@@ -285,10 +285,10 @@ install python/bindir/cloud-external-ipallocator.py ${RPM_BUILD_ROOT}%{_bindir}/
 install -D client/target/pythonlibs/jasypt-1.9.2.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/jasypt-1.9.2.jar
 
 install -D packaging/centos7/cloud-ipallocator.rc ${RPM_BUILD_ROOT}%{_initrddir}/%{name}-ipallocator
-install -D packaging/centos7/cloud-management.sysconfig ${RPM_BUILD_ROOT}%{_sysconfdir}/sysconfig/%{name}-management
-install -D server/target/conf/cloudstack-sudoers ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management
-install -D packaging/centos7/cloud-management.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-management.service
 install -D packaging/centos7/cloud.limits ${RPM_BUILD_ROOT}%{_sysconfdir}/security/limits.d/cloud
+install -D packaging/systemd/cloudstack-management.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-management.service
+install -D packaging/systemd/cloudstack-management.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-management
+install -D server/target/conf/cloudstack-sudoers ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management
 touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid
 install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina
 
@@ -309,6 +309,7 @@ mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/agent
 mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
 mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/plugins
 install -D packaging/systemd/cloudstack-agent.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-agent.service
+install -D packaging/systemd/cloudstack-agent.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-agent
 install -D agent/target/transformed/agent.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/agent.properties
 install -D agent/target/transformed/environment.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/environment.properties
 install -D agent/target/transformed/log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/log4j-cloud.xml
@@ -329,6 +330,7 @@ install -D usage/target/transformed/db.properties ${RPM_BUILD_ROOT}%{_sysconfdir
 install -D usage/target/transformed/log4j-cloud_usage.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage/log4j-cloud.xml
 cp usage/target/dependencies/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib/
 install -D packaging/systemd/cloudstack-usage.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-usage.service
+install -D packaging/systemd/cloudstack-usage.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-usage
 mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/usage/
 
 # CLI
@@ -403,6 +405,10 @@ if [ ! -f %{_datadir}/cloudstack-common/scripts/vm/hypervisor/xenserver/vhd-util
     echo %{_datadir}/cloudstack-common/scripts/vm/hypervisor/xenserver/
 fi
 
+if [ -f %{_sysconfdir}/sysconfig/%{name}-management ] ; then
+    mv %{_sysconfdir}/sysconfig/%{name}-management  %{_sysconfdir}/default/%{name}-management
+fi
+
 %preun agent
 /sbin/service cloudstack-agent stop || true
 if [ "$1" == "0" ] ; then
@@ -482,7 +488,7 @@ pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
 %dir %attr(0770,root,cloud) %{_localstatedir}/cache/%{name}/management/work
 %dir %attr(0770,root,cloud) %{_localstatedir}/cache/%{name}/management/temp
 %dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/management
-%config(noreplace) %{_sysconfdir}/sysconfig/%{name}-management
+%config(noreplace) %{_sysconfdir}/default/%{name}-management
 %config(noreplace) %{_sysconfdir}/sudoers.d/%{name}-management
 %config(noreplace) %{_sysconfdir}/security/limits.d/cloud
 %config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/management/db.properties
@@ -533,6 +539,7 @@ pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
 %attr(0755,root,root) %{_bindir}/%{name}-agent-upgrade
 %attr(0755,root,root) %{_bindir}/%{name}-ssh
 %attr(0644,root,root) %{_unitdir}/%{name}-agent.service
+%config(noreplace) %{_sysconfdir}/default/%{name}-agent
 %attr(0644,root,root) %{_sysconfdir}/profile.d/%{name}-agent-profile.sh
 %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-agent
 %attr(0755,root,root) %{_datadir}/%{name}-common/scripts/network/cisco
@@ -559,6 +566,7 @@ pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
 
 %files usage
 %attr(0644,root,root) %{_unitdir}/%{name}-usage.service
+%config(noreplace) %{_sysconfdir}/default/%{name}-usage
 %attr(0644,root,root) %{_datadir}/%{name}-usage/*.jar
 %attr(0644,root,root) %{_datadir}/%{name}-usage/lib/*.jar
 %dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/usage