Skip to content

Commit

Permalink
AMBARI-21729 Owner of yarn.include stays to be root:root where as dfs…
Browse files Browse the repository at this point in the history 
….include gets changed to the correct one upon restart services (dsen)
  • Loading branch information
@d0zen1
d0zen1 committed Aug 16, 2017
1 parent ab8d58b commit 5c78bd7
Show file tree
Hide file tree
Showing 11 changed files with 4,880 additions and 15 deletions.
5 changes: 2 additions & 3 deletions ...-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,9 +169,6 @@
rm_host = rm_hosts[0]
rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1]
rm_https_port = default('/configurations/yarn-site/yarn.resourcemanager.webapp.https.address', ":8090").split(':')[-1]
# TODO UPGRADE default, update site during upgrade
rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
rm_nodes_exclude_dir = os.path.dirname(rm_nodes_exclude_path)

java64_home = config['hostLevelParams']['java_home']
java_exec = format("{java64_home}/bin/java")
Expand Down Expand Up @@ -241,13 +238,15 @@
#exclude file
exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
rm_nodes_exclude_dir = os.path.dirname(exclude_file_path)

nm_hosts = default("/clusterHostInfo/nm_hosts", [])
#incude file
include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None)
include_hosts = None
manage_include_files = default("/configurations/yarn-site/manage.include.files", False)
if include_file_path and manage_include_files:
rm_nodes_include_dir = os.path.dirname(include_file_path)
include_hosts = list(set(nm_hosts) - set(exclude_hosts))

ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))
Expand Down
14 changes: 13 additions & 1 deletion ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,10 +390,22 @@ def setup_resourcemanager():
create_parents=True,
cd_access='a',
)
File(params.rm_nodes_exclude_path,
File(params.exclude_file_path,
content=Template("exclude_hosts_list.j2"),
owner=params.yarn_user,
group=params.user_group
)
if params.include_hosts:
Directory(params.rm_nodes_include_dir,
mode=0755,
create_parents=True,
cd_access='a',
)
File(params.include_file_path,
content=Template("include_hosts_list.j2"),
owner=params.yarn_user,
group=params.user_group
)
File(params.yarn_job_summary_log,
owner=params.yarn_user,
group=params.user_group
Expand Down
5 changes: 2 additions & 3 deletions ...-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,9 +166,6 @@
rm_host = rm_hosts[0]
rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1]
rm_https_port = default('/configurations/yarn-site/yarn.resourcemanager.webapp.https.address', ":8090").split(':')[-1]
# TODO UPGRADE default, update site during upgrade
rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
rm_nodes_exclude_dir = os.path.dirname(rm_nodes_exclude_path)

java64_home = config['hostLevelParams']['java_home']
java_exec = format("{java64_home}/bin/java")
Expand Down Expand Up @@ -238,13 +235,15 @@
#exclude file
exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
rm_nodes_exclude_dir = os.path.dirname(exclude_file_path)

nm_hosts = default("/clusterHostInfo/nm_hosts", [])
#incude file
include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None)
include_hosts = None
manage_include_files = default("/configurations/yarn-site/manage.include.files", False)
if include_file_path and manage_include_files:
rm_nodes_include_dir = os.path.dirname(include_file_path)
include_hosts = list(set(nm_hosts) - set(exclude_hosts))

ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))
Expand Down
14 changes: 13 additions & 1 deletion ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,10 +391,22 @@ def setup_resourcemanager():
create_parents=True,
cd_access='a',
)
File(params.rm_nodes_exclude_path,
File(params.exclude_file_path,
content=Template("exclude_hosts_list.j2"),
owner=params.yarn_user,
group=params.user_group
)
if params.include_hosts:
Directory(params.rm_nodes_include_dir,
mode=0755,
create_parents=True,
cd_access='a',
)
File(params.include_file_path,
content=Template("include_hosts_list.j2"),
owner=params.yarn_user,
group=params.user_group
)
# This depends on the parent directory already existing.
File(params.yarn_job_summary_log,
owner=params.yarn_user,
Expand Down
160 changes: 158 additions & 2 deletions ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,28 @@ def test_configure_default(self):
self.assert_configure_default()
self.assertNoMoreResources()

def test_configure_default_with_include_file_dont_manage(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="configure",
config_file="default_yarn_include_file_dont_manage.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_default(is_include_file_configured=True, manage_include_files=False)
self.assertNoMoreResources()

def test_configure_default_with_include_file_manage(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="configure",
config_file="default_yarn_include_file_manage.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_default(is_include_file_configured=True, manage_include_files=True)
self.assertNoMoreResources()

def test_start_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
Expand Down Expand Up @@ -100,6 +122,28 @@ def test_configure_secured(self):
)
self.assert_configure_secured()

def test_configure_secured_with_include_file_dont_manage(self):

self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="configure",
config_file="secured_yarn_include_file_dont_manage.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_secured(is_include_file_configured=True, manage_include_files=False)

def test_configure_secured_with_include_file_manage(self):

self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="configure",
config_file="secured_yarn_include_file_manage.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_secured(is_include_file_configured=True, manage_include_files=True)

def test_start_secured(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
Expand Down Expand Up @@ -179,7 +223,95 @@ def test_decommission_secured(self):

self.assertNoMoreResources()

def assert_configure_default(self):
def test_decommission_default_with_include_file_dont_manage(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname = "Resourcemanager",
command = "decommission",
config_file="default_yarn_include_file_dont_manage.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
content = Template('exclude_hosts_list.j2'),
group = 'hadoop',
)
self.assertResourceCalled('Execute', ' yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
user = 'yarn',
)
self.assertNoMoreResources()

def test_decommission_default_with_include_file_manage(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname = "Resourcemanager",
command = "decommission",
config_file="default_yarn_include_file_manage.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
content = Template('exclude_hosts_list.j2'),
group = 'hadoop',
)
self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include',
owner = 'yarn',
content = Template('include_hosts_list.j2'),
group = 'hadoop',
)
self.assertResourceCalled('Execute', ' yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
user = 'yarn',
)
self.assertNoMoreResources()

def test_decommission_secured_with_include_file_dont_manage(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname = "Resourcemanager",
command = "decommission",
config_file="secured_yarn_include_file_dont_manage.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
content = Template('exclude_hosts_list.j2'),
group = 'hadoop',
)
self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/rm.service.keytab rm/c6401.ambari.apache.org@EXAMPLE.COM; yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
user = 'yarn',
)

self.assertNoMoreResources()

def test_decommission_secured_with_include_file_manage(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname = "Resourcemanager",
command = "decommission",
config_file="secured_yarn_include_file_manage.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
content = Template('exclude_hosts_list.j2'),
group = 'hadoop',
)
self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include',
owner = 'yarn',
content = Template('include_hosts_list.j2'),
group = 'hadoop',
)
self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/rm.service.keytab rm/c6401.ambari.apache.org@EXAMPLE.COM; yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
user = 'yarn',
)

self.assertNoMoreResources()

def assert_configure_default(self, is_include_file_configured = False, manage_include_files=False):
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
group = 'hadoop',
Expand Down Expand Up @@ -235,9 +367,21 @@ def assert_configure_default(self):
cd_access = 'a',
)
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
content = Template('exclude_hosts_list.j2'),
owner = 'yarn',
group = 'hadoop',
)
if is_include_file_configured and manage_include_files:
self.assertResourceCalled('Directory', '/etc/hadoop/conf_for_include',
mode = 0755,
create_parents = True,
cd_access = 'a',
)
self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include',
content = Template('include_hosts_list.j2'),
owner = 'yarn',
group = 'hadoop',
)
self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
owner = 'yarn',
group = 'hadoop',
Expand Down Expand Up @@ -347,7 +491,7 @@ def assert_configure_default(self):
group = 'hadoop',
)

def assert_configure_secured(self):
def assert_configure_secured(self, is_include_file_configured = False, manage_include_files=False):
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
group = 'hadoop',
Expand Down Expand Up @@ -403,9 +547,21 @@ def assert_configure_secured(self):
cd_access = 'a',
)
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
content = Template('exclude_hosts_list.j2'),
owner = 'yarn',
group = 'hadoop',
)
if is_include_file_configured and manage_include_files:
self.assertResourceCalled('Directory', '/etc/hadoop/conf_for_include',
mode = 0755,
create_parents = True,
cd_access = 'a',
)
self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include',
content = Template('include_hosts_list.j2'),
owner = 'yarn',
group = 'hadoop',
)
self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
owner = 'yarn',
group = 'hadoop',
Expand Down
Loading

0 comments on commit 5c78bd7

Please sign in to comment.