openstack
/
sahara
Code Issues Proposed changes

Made Ambari RPM location configurable 

The Ambari RPM location is now an ambari configuration property in
the blueprint.  This will support debug/issue resolution scenarios
as well as the export of this information.

Fixes:  bug #1208966

Change-Id: Ie0bfbd39d687c8c26d654b7fcfb369f764d48ef2
(cherry picked from commit f9a7b20908)
This commit is contained in:
Jon Maron 2013-08-06 15:10:37 -04:00 committed by Sergey Lukjanov
parent bbc7b27209
commit b26e5e080b
7 changed files with 70 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
savanna/plugins/hdp/ambariplugin.py
View File

@ -50,11 +50,13 @@ class AmbariPlugin(p.ProvisioningPluginBase):
ambari_host = self._determine_host_for_server_component(
'AMBARI_SERVER', cluster_spec, hosts)
self.cluster_name_to_ambari_host_mapping[cluster.name] = ambari_host
rpm = self._get_rpm_uri(cluster_spec)
servers = []
for host in hosts:
servers.append(
h.HadoopServer(host, cluster_spec.node_groups[host.role]))
h.HadoopServer(host, cluster_spec.node_groups[host.role],
ambari_rpm=rpm))
provisioned = self._provision_cluster(cluster.name, cluster_spec,
ambari_host, servers)
@ -591,3 +593,7 @@ class AmbariPlugin(p.ProvisioningPluginBase):
def validate(self, cluster):
validator = v.Validator()
validator.validate(cluster)
def _get_rpm_uri(self, cluster_spec):
ambari_config = cluster_spec.configurations['ambari']
return ambari_config.get('rpm', None)

12
savanna/plugins/hdp/hadoopserver.py
View File

@ -16,16 +16,22 @@
import re
from savanna.openstack.common import log as logging
AMBARI_RPM = 'http://s3.amazonaws.com/' \
'dev.hortonworks.com/AMBARI.b6-1.x/' \
'repos/centos6/AMBARI.b6-1.x-1.el6.' \
'noarch.rpm'
LOG = logging.getLogger(__name__)
class HadoopServer:
_master_ip = None
def __init__(self, instance, node_group):
def __init__(self, instance, node_group, ambari_rpm=None):
self.instance = instance
self.node_group = node_group
self._remote = self.instance.remote
self.ambari_rpm = ambari_rpm or AMBARI_RPM
def provision_ambari(self, ambari_server_ip):
self.install_rpms()
@ -40,9 +46,7 @@ class HadoopServer:
"{0}: Installing rpm's ...".format(self.instance.hostname))
#TODO(jspeidel): based on image type, use correct command
rpm_cmd = 'rpm -Uvh http://s3.amazonaws.com/dev.hortonworks' \
'.com/AMBARI.b6-1.x/repos/centos6/AMBARI.b6-1.x-1.el6' \
'.noarch.rpm'
rpm_cmd = 'rpm -Uvh ' + self.ambari_rpm
self._remote.execute_command(rpm_cmd)
self._remote.execute_command('yum -y install epel-release')

18
savanna/plugins/hdp/resources/ambari-config-resource.json
View File

@ -1733,6 +1733,24 @@
"name": "hadoop_heapsize",
"scope": "cluster"
},
{
"applicable_target": "general",
"config_type": "string",
"default_value": "http://s3.amazonaws.com/dev.hortonworks.com/AMBARI.b6-1.x/repos/centos6/AMBARI.b6-1.x-1.el6.noarch.rpm",
"description": "...",
"is_optional": true,
"name": "rpm",
"scope": "cluster"
},
{
"applicable_target": "general",
"config_type": "int",
"default_value": 8080,
"description": "...",
"is_optional": true,
"name": "port",
"scope": "cluster"
},
{
"applicable_target": "HDFS",
"config_type": "string",

7
savanna/plugins/hdp/resources/default-cluster.template
View File

@ -369,6 +369,13 @@
{ "name" : "dfs.https.address", "value" : "%AMBARI_HOST%:50470" },
{ "name" : "dfs.datanode.data.dir.perm", "value" : "750" }
]
},
{
"name" : "ambari",
"properties" : [
{ "name" : "port", "value" : "8080" },
{ "name" : "rpm", "value" : "http://s3.amazonaws.com/dev.hortonworks.com/AMBARI.b6-1.x/repos/centos6/AMBARI.b6-1.x-1.el6.noarch.rpm" }
]
}
]
}

7
savanna/tests/unit/plugins/hdp/ambariplugin_test.py
View File

@ -74,13 +74,6 @@ class AmbariPluginTest(unittest2.TestCase):
for node_group in cluster.node_groups:
self.assertEquals(cluster.default_image_id, node_group.image)
def test_get_configs(self):
plugin = ap.AmbariPlugin()
configs = plugin.get_configs("1.3.0")
self.assertEqual(786, len(configs),
"wrong number of configuration properties")
def test__set_ambari_credentials__admin_only(self):
self.requests = []
plugin = ap.AmbariPlugin()

47
savanna/tests/unit/plugins/hdp/clusterspec_test.py
View File

@ -13,8 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import pkg_resources as pkg
from savanna.plugins.hdp import clusterspec as cs
from savanna import version
import unittest2
@ -22,12 +23,10 @@ class ClusterSpecTest(unittest2.TestCase):
service_validators = {}
#TODO(jspeidel): test host manifest
def test_parse_default_with_hosts(self):
with open(os.path.join(os.path.realpath('../plugins'), 'hdp',
'resources',
'default-cluster.template'), 'r') as f:
cluster_config_file = f.read()
cluster_config_file = pkg.resource_string(
version.version_info.package,
'plugins/hdp/resources/default-cluster.template')
servers = []
server1 = TestServer('host1', 'master', '11111', 3, '111.11.1111',
@ -55,10 +54,9 @@ class ClusterSpecTest(unittest2.TestCase):
return cluster_config
def test_select_correct_server_for_ambari_host(self):
with open(os.path.join(os.path.realpath('../plugins'), 'hdp',
'resources',
'default-cluster.template'), 'r') as f:
cluster_config_file = f.read()
cluster_config_file = pkg.resource_string(
version.version_info.package,
'plugins/hdp/resources/default-cluster.template')
servers = []
server1 = TestServer('ambari_machine', 'master', '11111', 3,
@ -86,10 +84,9 @@ class ClusterSpecTest(unittest2.TestCase):
'Ambari host not found')
def test_parse_default(self):
with open(os.path.join(os.path.realpath('../plugins'), 'hdp',
'resources',
'default-cluster.template'), 'r') as f:
cluster_config_file = f.read()
cluster_config_file = pkg.resource_string(
version.version_info.package,
'plugins/hdp/resources/default-cluster.template')
cluster_config = cs.ClusterSpec(cluster_config_file)
@ -99,11 +96,22 @@ class ClusterSpecTest(unittest2.TestCase):
return cluster_config
def test_ambari_rpm(self):
cluster_config_file = pkg.resource_string(
version.version_info.package,
'plugins/hdp/resources/default-cluster.template')
cluster_config = cs.ClusterSpec(cluster_config_file)
self._assert_configurations(cluster_config.configurations)
ambari_config = cluster_config.configurations['ambari']
self.assertIsNotNone('no rpm uri found',
ambari_config.get('rpm', None))
def test_normalize(self):
with open(os.path.join(os.path.realpath('../plugins'), 'hdp',
'resources',
'default-cluster.template'), 'r') as f:
cluster_config_file = f.read()
cluster_config_file = pkg.resource_string(
version.version_info.package,
'plugins/hdp/resources/default-cluster.template')
cluster_config = cs.ClusterSpec(cluster_config_file)
cluster = cluster_config.normalize()
@ -282,11 +290,12 @@ class ClusterSpecTest(unittest2.TestCase):
self.assertEquals(cardinality, component.cardinality)
def _assert_configurations(self, configurations):
self.assertEquals(4, len(configurations))
self.assertEqual(5, len(configurations))
self.assertIn('global', configurations)
self.assertIn('core-site', configurations)
self.assertIn('mapred-site', configurations)
self.assertIn('hdfs-site', configurations)
self.assertIn('ambari', configurations)
def _assert_host_role_mappings(self, node_groups):
self.assertEquals(2, len(node_groups))

2
setup.py
View File

@ -43,6 +43,8 @@ setuptools.setup(
packages=setuptools.find_packages(exclude=['bin']),
package_data={'savanna': [
'plugins/vanilla/resources/*.xml',
'plugins/hdp/resources/*.json',
'plugins/hdp/resources/*.template',
'swift/resources/*.xml',
'tests/unit/resources/*.xml',
'tests/unit/resources/*.txt',