Merge pull request #344 from caskdata/feature/remove-hdp-version-attrs

Feature/remove hdp version attrs

Author: dereklwood

attributes/default.rb

@@ -99,54 +99,6 @@
 end
 
 # MapReduce settings
-full_version =
-  case node['hadoop']['distribution_version']
-  when '2.2.0.0'
-    '2.2.0.0-2041'
-  when '2.2.1.0'
-    '2.2.1.0-2340'
-  when '2.2.4.2'
-    '2.2.4.2-2'
-  when '2.2.4.4'
-    '2.2.4.4-16'
-  when '2.2.6.0'
-    '2.2.6.0-2800'
-  when '2.2.6.3'
-    '2.2.6.3-1'
-  when '2.2.8.0'
-    '2.2.8.0-3150'
-  when '2.2.9.0'
-    '2.2.9.0-3393'
-  when '2.3.0.0'
-    '2.3.0.0-2557'
-  when '2.3.2.0'
-    '2.3.2.0-2950'
-  when '2.3.4.0'
-    '2.3.4.0-3485'
-  when '2.3.4.7'
-    '2.3.4.7-4'
-  when '2.3.6.0'
-    '2.3.6.0-3796'
-  when '2.4.0.0'
-    '2.4.0.0-169'
-  when '2.4.2.0'
-    '2.4.2.0-258'
-  when '2.4.3.0'
-    '2.4.3.0-227'
-  when '2.5.0.0'
-    '2.5.0.0-1245'
-  when '2.5.3.0'
-    '2.5.3.0-37'
-  when '2.5.5.0'
-    '2.5.5.0-157'
-  when '2.6.0.3'
-    '2.6.0.3-8'
-  when '2.6.1.0'
-    '2.6.1.0-129'
-  else
-    node['hadoop']['distribution_version']
-  end
-
 default['hadoop']['hadoop_env']['hadoop_opts'] = '-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}'
 default['hadoop']['mapred_env']['hadoop_opts'] = '-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}'
 
@@ -158,8 +110,9 @@
 
   distro = node['hadoop']['distribution']
   lzo_jar = distro == 'hdp' ? 'hadoop-lzo-0.6.0.${hdp.version}.jar' : 'hadoop-lzo-0.5.1.jar'
-  default['hadoop']['hadoop_env']['hadoop_opts'] += " -D#{distro}.version=#{full_version}"
-  default['hadoop']['mapred_env']['hadoop_opts'] += " -D#{distro}.version=#{full_version}"
+  # %{_FULL_VERSION} will be interpolated in the default recipe, when helper libraries are available
+  default['hadoop']['hadoop_env']['hadoop_opts'] += " -D#{distro}.version=%<_FULL_VERSION>s"
+  default['hadoop']['mapred_env']['hadoop_opts'] += " -D#{distro}.version=%<_FULL_VERSION>s"
   default['hadoop']['mapred_site']['mapreduce.admin.map.child.java.opts'] =
     "-server -Djava.net.preferIPv4Stack=true -D#{distro}.version=${#{distro}.version}"
   default['hadoop']['mapred_site']['mapreduce.admin.user.env'] =

recipes/default.rb

@@ -162,6 +162,14 @@
     # rubocop:enable Style/Next
   end
 
+  # Evaluate any Delayed Interpolation tokens in *-env attributes
+  delayed_attrs = { _FULL_VERSION: hdp_version }
+  if node['hadoop'].key?(envfile) && !node['hadoop'][envfile].empty?
+    node['hadoop'][envfile].each do |k, v|
+      node.default['hadoop'][envfile][k] = v % delayed_attrs
+    end
+  end
+
   template "#{hadoop_conf_dir}/#{envfile.tr('_', '-')}.sh" do
     source 'generic-env.sh.erb'
     mode '0755'

spec/default_spec.rb

@@ -86,6 +86,18 @@
       )
     end
 
+    it 'renders file hadoop-env.sh with HADOOP_OPTS containing the correct hdp.version' do
+      expect(chef_run).to render_file('/etc/hadoop/conf.chef/hadoop-env.sh').with_content(
+        /HADOOP_OPTS.*2.3.4.7-4/
+      )
+    end
+
+    it 'renders file mapred-env.sh with HADOOP_OPTS containing the correct hdp.version' do
+      expect(chef_run).to render_file('/etc/hadoop/conf.chef/mapred-env.sh').with_content(
+        /HADOOP_OPTS.*2.3.4.7-4/
+      )
+    end
+
     it 'renders file hadoop-policy.xml with test.property' do
       expect(chef_run).to render_file('/etc/hadoop/conf.chef/hadoop-policy.xml').with_content(
         /test.property/