Merge pull request #397 from dshulyak/multi_transports

rsync + raw_ssh by default

Dockerfile

@@ -7,9 +7,8 @@ ADD resources /resources
 ADD templates /templates
 ADD run.sh /run.sh
 
-RUN apt-get update
+RUN apt-get upgrade && apt-get update
-RUN apt-get install -y python python-dev python-distribute python-pip \
+RUN apt-get install -y python python-dev python-distribute python-pip openssh-client rsync libyaml-dev vim libffi-dev libssl-dev git
-    libyaml-dev vim libffi-dev libssl-dev git
 RUN pip install ansible
 
 RUN pip install git+https://github.com/Mirantis/solar.git

bootstrap/playbooks/build-main.yaml

@@ -3,10 +3,12 @@
 - name: Main build script
   hosts: all
   sudo: yes
+  vars:
+    ssh_ip_mask: "10.0.0.*"
   tasks:
     - include: tasks/base.yaml
     - include: tasks/puppet.yaml
     - include: tasks/docker.yaml
-    #- include: celery.yaml tags=['master'] celery_dir=/var/run/celery
     - include: tasks/cloud_archive.yaml
-    #- include: tasks/mos.yaml
+    - include: tasks/ssh_conf.yaml
+

bootstrap/playbooks/celery.yaml

@@ -18,6 +18,4 @@
     - shell: celery multi start 2 -A solar.orchestration.runner -P:2 prefork -c:1 1 -c:2 3 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }}
              chdir={{ celery_dir }}
       tags: [master]
-    - shell: celery multi start 1 -A solar.orchestration.runner -Q:1  {{ hostname.stdout }}
+
-             chdir={{ celery_dir }}
-      tags: [slave]

bootstrap/playbooks/files/ssh_conf

@@ -0,0 +1,2 @@
+Host {{ssh_ip_mask}}
+  StrictHostKeyChecking no

bootstrap/playbooks/solar.yaml

@@ -2,12 +2,15 @@
 
 - hosts: all
   sudo: yes
+  vars:
+    ssh_ip_mask: "10.0.0.*"
   tasks:
     # upgrade pbr first, old version throws strange errors
     - shell: pip install pbr -U
     # Setup development env for solar
     - shell: pip install -e . chdir=/vagrant
     - shell: pip install git+git://github.com/Mirantis/solar-agent.git
+    - include: tasks/ssh_conf.yaml
 
 - hosts: all
   tasks:

bootstrap/playbooks/tasks/ssh_conf.yaml

@@ -0,0 +1,3 @@
+---
+
+- template: src=files/ssh_conf dest=/root/.ssh/config

doc/source/transports.rst

@@ -31,6 +31,16 @@ Currently there are following sync transports available:
 * solar_agent
 * torrent
 
+Ssh host key checking
+---------------------
+Solar wont disable strict host key checking by default, so before working
+with solar ensure that strict host key checking is disabled, or all target hosts added to .ssh/known_hosts file.
+
+Example of .ssh/config ::
+
+  Host 10.0.0.*
+    StrictHostKeyChecking no
+
 Run transport
 -------------
 

docker-compose.yml

@@ -8,6 +8,8 @@ solar-celery:
     - /vagrant/templates:/vagrant/templates
     - /vagrant/resources:/vagrant/resources
     - /vagrant/library:/vagrant/library
+    - ~/.ssh:/root/.ssh
+    - ./bootstrap/playbooks/celery.yaml:/celery.yaml
   environment:
     - REDIS_HOST=redis
     - REDIS_PORT=6379

examples/hosts_file/hosts.py

@@ -10,12 +10,12 @@ from solar.dblayer.model import ModelMeta
 def run():
     ModelMeta.remove_all()
 
-    resources = vr.create('nodes', 'templates/nodes_with_transports.yaml', {'count': 2})
+    resources = vr.create('nodes', 'templates/nodes.yaml', {'count': 2})
-    nodes = [x for x in resources if x.name.startswith('node')]
+
-    node1, node2 = nodes
+    node1, node2 = [x for x in resources if x.name.startswith('node')]
+    hosts1, hosts2 = [x for x in resources
+                      if x.name.startswith('hosts_file')]
 
-    hosts1 = vr.create('hosts_file1', 'resources/hosts_file', {})[0]
-    hosts2 = vr.create('hosts_file2', 'resources/hosts_file', {})[0]
     node1.connect(hosts1, {
         'name': 'hosts:name',
         'ip': 'hosts:ip',
@@ -36,5 +36,4 @@ def run():
         'ip': 'hosts:ip',
     })
 
-
 run()

resources/transport_rsync/meta.yaml

@@ -0,0 +1,23 @@
+id: transport_rsync
+input:
+  key:
+    schema: str!
+    value:
+  user:
+    schema: str!
+    value:
+  name:
+    schema: str!
+    value: rsync
+  location_id:
+    schema: str
+    value:
+    reverse: True
+    is_own: False
+  transports_id:
+    schema: str
+    value:
+    is_emit: False
+  port:
+    schema: int
+    value: 3579

run.sh

@@ -6,6 +6,6 @@ if [ -d /solar ]; then
 fi
 
 #used only to start celery on docker
-ansible-playbook -v -i "localhost," -c local /celery.yaml --skip-tags slave,stop
+ansible-playbook -v -i "localhost," -c local /celery.yaml --skip-tags install
 
 tail -f /var/run/celery/*.log

solar/core/handlers/ansible_template.py

@@ -19,7 +19,6 @@ import os
 from solar.core.handlers.base import SOLAR_TEMP_LOCAL_LOCATION
 from solar.core.handlers.base import TempFileHandler
 from solar.core.log import log
-from solar import errors
 
 # otherwise fabric will sys.exit(1) in case of errors
 env.warn_only = True
@@ -52,15 +51,8 @@ class AnsibleTemplate(TempFileHandler):
                      '-i', remote_inventory_file, remote_playbook_file]
         log.debug('EXECUTING: %s', ' '.join(call_args))
 
-        out = self.transport_run.run(resource, *call_args)
+        rst = self.transport_run.run(resource, *call_args)
-        log.debug(out)
+        self.verify_run_result(call_args, rst)
-        if out.failed:
-            raise errors.SolarError(out)
-
-        # with fabric_api.shell_env(ANSIBLE_HOST_KEY_CHECKING='False'):
-        #     out = fabric_api.local(' '.join(call_args), capture=True)
-        # if out.failed:
-        #     raise errors.SolarError(out)
 
     def _create_inventory(self, r):
         directory = self.dirs[r.name]

solar/core/handlers/base.py

@@ -21,6 +21,7 @@ import tempfile
 from solar.core.log import log
 from solar.core.transports.ssh import SSHRunTransport
 from solar.core.transports.ssh import SSHSyncTransport
+from solar import errors
 from solar import utils
 
 
@@ -42,6 +43,13 @@ class BaseHandler(object):
         self.transport_sync.bind_with(self.transport_run)
         self.transport_run.bind_with(self.transport_sync)
 
+    def verify_run_result(self, cmd, result):
+        rc, out, err = result
+        log.debug('CMD %r RC %s OUT %s ERR %s', cmd, rc, out, err)
+        if rc:
+            message = 'CMD %r failed RC %s ERR %s' % (cmd, rc, err)
+            raise errors.SolarError(message)
+
     def __enter__(self):
         return self
 

solar/core/handlers/puppet.py

@@ -46,7 +46,7 @@ class Puppet(TempFileHandler):
             cmd_args.append('--modulepath={}'.format(
                 resource.args['puppet_modules']))
 
-        cmd = self.transport_run.run(
+        rc, out, err = self.transport_run.run(
             resource,
             *cmd_args,
             env={
@@ -55,12 +55,12 @@ class Puppet(TempFileHandler):
             use_sudo=True,
             warn_only=True
         )
+        log.debug('CMD %r RC %s OUT %s ERR %s', cmd_args, rc, out, err)
         # 0 - no changes, 2 - successfull changes
-        if cmd.return_code not in [0, 2]:
+        if rc not in [0, 2]:
             raise errors.SolarError(
-                'Puppet for {} failed with {}'.format(
+                'Puppet for {} failed with RC {}'.format(
-                    resource.name, cmd.return_code))
+                    resource.name, rc))
-        return cmd
 
     def _make_args(self, resource):
         return {resource.name: {'input': resource.args}}

solar/core/handlers/shell.py

@@ -18,7 +18,6 @@ import os
 from solar.core.handlers.base import SOLAR_TEMP_LOCAL_LOCATION
 from solar.core.handlers.base import TempFileHandler
 from solar.core.log import log
-from solar import errors
 
 
 class Shell(TempFileHandler):
@@ -36,15 +35,10 @@ class Shell(TempFileHandler):
         self.transport_sync.copy(resource, self.dst, '/tmp')
         self.transport_sync.sync_all()
 
-        cmd = self.transport_run.run(
+        rst = self.transport_run.run(
             resource,
             'bash', action_file_name,
             use_sudo=True,
             warn_only=True
         )
-
+        self.verify_run_results(['bash', action_file_name], rst)
-        if cmd.return_code:
-            raise errors.SolarError(
-                'Bash execution for {} failed with {}'.format(
-                    resource.name, cmd.return_code))
-        return cmd

solar/core/resource/virtual_resource.py

@@ -266,14 +266,17 @@ def parse_list_input(r_input, args):
     connections = []
     assignments = {}
     for arg in args:
-        if is_connection(arg):
+        if isinstance(arg, dict):
+            n_connections, n_assign = parse_dict_input(
+                r_input, arg)
+            connections.extend(n_connections)
+            if n_assign:
+                add_assignment(assignments, r_input, n_assign)
+        elif is_connection(arg):
             c = parse_connection(r_input, arg)
             connections.append(c)
         else:
-            try:
+            add_assignment(assignments, r_input, arg)
-                assignments[r_input].append(arg)
-            except KeyError:
-                assignments[r_input] = [arg]
     return connections, assignments
 
 
@@ -293,6 +296,13 @@ def parse_dict_input(r_input, args):
     return connections, assignments
 
 
+def add_assignment(assignments, r_input, arg):
+    try:
+        assignments[r_input].append(arg)
+    except KeyError:
+        assignments[r_input] = [arg]
+
+
 def is_connection(arg):
     if isinstance(arg, basestring) and '::' in arg:
         return True

solar/core/transports/base.py

@@ -12,6 +12,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from solar.core.log import log
+from solar import errors
+
 
 class Executor(object):
 
@@ -38,7 +41,13 @@ class Executor(object):
 
     def run(self, transport):
         if self.valid:
-            self._executor(transport)
+            result = self._executor(transport)
+            if isinstance(result, tuple) and len(result) == 3:
+                # TODO Include file information in result
+                rc, out, err = result
+                log.debug('RC %s OUT %s ERR %s', rc, out, err)
+                if rc:
+                    raise errors.SolarError(err)
 
 
 class SolarRunResultWrp(object):

solar/core/transports/bat.py

@@ -17,8 +17,8 @@ from solar.core.transports.base import RunTransport
 from solar.core.transports.base import SolarTransport
 from solar.core.transports.base import SyncTransport
 from solar.core.transports.rsync import RsyncSyncTransport
-from solar.core.transports.ssh import SSHRunTransport
 from solar.core.transports.ssh import SSHSyncTransport
+from solar.core.transports.ssh_raw import RawSSHRunTransport
 try:
     from solar.core.transports.solar_agent_transport import SolarAgentRunTransport  # NOQA
     from solar.core.transports.solar_agent_transport import SolarAgentSyncTransport  # NOQA
@@ -42,7 +42,7 @@ KNOWN_SYNC_TRANSPORTS = {
 
 
 KNOWN_RUN_TRANSPORTS = {
-    'ssh': SSHRunTransport
+    'ssh': RawSSHRunTransport
 }
 
 

solar/core/transports/rsync.py

@@ -12,11 +12,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from fabric import api as fabric_api
-
 from solar.core.log import log
 from solar.core.transports.base import Executor
 from solar.core.transports.base import SyncTransport
+from solar.utils import execute
 
 # XXX:
 # currently we don't support key verification or acceptation
@@ -55,9 +54,8 @@ class RsyncSyncTransport(SyncTransport):
                          _from=_from,
                          _to=_to)
 
-        rsync_executor = lambda transport: fabric_api.local(
+        rsync_executor = lambda transport: execute(
-            rsync_cmd
+            rsync_cmd, shell=True)
-        )
 
         log.debug("RSYNC CMD: %r" % rsync_cmd)
 

solar/core/transports/ssh_raw.py

@@ -12,27 +12,31 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from fabric import api as fabric_api
 
 from solar.core.log import log
 from solar.core.transports.base import RunTransport
+from solar.utils import execute
 
 
 class _RawSSHTransport(object):
 
-    def _ssh_props(self, resource):
+    def settings(self, resource):
-        return {
+        transport = self.get_transport_data(resource)
-            'ssh_key': resource.args['ssh_key'].value,
+        host = resource.ip()
-            'ssh_user': resource.args['ssh_user'].value
+        user = transport['user']
-        }
+        port = transport['port']
+        key = transport['key']
+        return {'ssh_user': user,
+                'ssh_key': key,
+                'port': port,
+                'ip': host}
 
-    def _ssh_command_host(self, resource):
+    def _ssh_command_host(self, settings):
-        return '{}@{}'.format(resource.args['ssh_user'].value,
+        return '{}@{}'.format(settings['ssh_user'],
-                              resource.args['ip'].value)
+                              settings['ip'])
 
-    def _ssh_cmd(self, resource):
+    def _ssh_cmd(self, settings):
-        props = self._ssh_props(resource)
+        return ('ssh', '-i', settings['ssh_key'])
-        return ('ssh', '-i', props['ssh_key'])
 
 
 class RawSSHRunTransport(RunTransport, _RawSSHTransport):
@@ -40,23 +44,30 @@ class RawSSHRunTransport(RunTransport, _RawSSHTransport):
     def run(self, resource, *args, **kwargs):
         log.debug("RAW SSH: %s", args)
 
-        cmds = []
+        commands = []
-        cwd = kwargs.get('cwd')
+        prefix = []
-        if cwd:
-            cmds.append(('cd', cwd))
-
-        cmds.append(args)
-
         if kwargs.get('use_sudo', False):
-            cmds = [('sudo', ) + cmd for cmd in cmds]
+            prefix.append('sudo')
 
-        cmds = [' '.join(cmd) for cmd in cmds]
+        if kwargs.get('cwd'):
+            cmd = prefix + ['cd', kwargs['cwd']]
+            commands.append(' '.join(cmd))
 
-        remote_cmd = '\"%s\"' % ' && '.join(cmds)
+        env = []
+        if 'env' in kwargs:
+            for key, value in kwargs['env'].items():
+                env.append('{}={}'.format(key, value))
 
-        ssh_cmd = self._ssh_cmd(resource)
+        cmd = prefix + env + list(args)
-        ssh_cmd += (self._ssh_command_host(resource), remote_cmd)
+        commands.append(' '.join(cmd))
 
-        log.debug("SSH CMD: %r", ssh_cmd)
+        remote_cmd = '\"%s\"' % ' && '.join(commands)
 
-        return fabric_api.local(' '.join(ssh_cmd))
+        settings = self.settings(resource)
+        ssh_cmd = self._ssh_cmd(settings)
+        ssh_cmd += (self._ssh_command_host(settings), remote_cmd)
+
+        log.debug("RAW SSH CMD: %r", ssh_cmd)
+        # TODO convert it to SolarRunResult
+
+        return execute(' '.join(ssh_cmd), shell=True)

solar/test/test_virtual_resource.py

@@ -202,6 +202,19 @@ def test_parse_connection_disable_events():
     assert correct_connection == connection
 
 
+def test_parse_list_of_connected_dicts():
+    inputs = {'list': [
+        {'key': 'emitter1::key'},
+        {'key': 'emitter2::key'}]}
+    connections, assignments = vr.parse_inputs(inputs)
+    assert assignments == {}
+    assert connections == [
+        {'child_input': 'list:key', 'parent_input': 'key',
+         'parent': 'emitter1', 'events': None},
+        {'child_input': 'list:key', 'parent_input': 'key',
+         'parent': 'emitter2', 'events': None}]
+
+
 def test_setting_location(tmpdir):
     # XXX: make helper for it
     base_path = os.path.join(

solar/utils.py

@@ -43,6 +43,17 @@ def communicate(command, data):
         stderr=subprocess.PIPE)
     return popen.communicate(input=data)[0]
 
+
+def execute(command, shell=False):
+    popen = subprocess.Popen(
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        shell=shell)
+    out, err = popen.communicate()
+    return popen.returncode, out, err
+
+
 # Configure jinja2 filters
 jinja_env_with_filters = Environment()
 jinja_env_with_filters.filters['to_json'] = to_json

templates/nodes.yaml

@@ -7,13 +7,23 @@ resources:
     values:
       ssh_user: 'vagrant'
       ssh_key: '/vagrant/.vagrant/machines/solar-dev{{j}}/virtualbox/private_key'
+  - id: rsync{{j}}
+    from: resources/transport_rsync
+    values:
+      user: vagrant
+      key: /vagrant/.vagrant/machines/solar-dev{{j}}/virtualbox/private_key
   - id: transports{{j}}
     from: resources/transports
     values:
-      transports:key: ssh_transport{{j}}::ssh_key
+      transports:
-      transports:user: ssh_transport{{j}}::ssh_user
+        - key: ssh_transport{{j}}::ssh_key
-      transports:port: ssh_transport{{j}}::ssh_port
+          user: ssh_transport{{j}}::ssh_user
-      transports:name: ssh_transport{{j}}::name
+          port: ssh_transport{{j}}::ssh_port
+          name: ssh_transport{{j}}::name
+        - key: rsync{{j}}::key
+          name: rsync{{j}}::name
+          user: rsync{{j}}::user
+          port: rsync{{j}}::port
   - id: node{{j}}
     from: resources/ro_node
     values:

templates/nodes_with_transports.yaml

@@ -6,13 +6,23 @@ resources:
     values:
       ssh_user: 'vagrant'
       ssh_key: '/vagrant/.vagrant/machines/solar-dev{{i + 1}}/virtualbox/private_key'
+  - id: rsync{{i}}
+    from: resources/transport_rsync
+    values:
+      user: vagrant
+      key: /vagrant/.vagrant/machines/solar-dev{{i + 1}}/virtualbox/private_key
   - id: transports{{i}}
     from: resources/transports
     values:
-      transports:key: ssh_transport{{i}}::ssh_key
+      transports:
-      transports:user: ssh_transport{{i}}::ssh_user
+        - key: ssh_transport{{i}}::ssh_key
-      transports:port: ssh_transport{{i}}::ssh_port
+          user: ssh_transport{{i}}::ssh_user
-      transports:name: ssh_transport{{i}}::name
+          port: ssh_transport{{i}}::ssh_port
+          name: ssh_transport{{i}}::name
+        - key: rsync{{i}}::key
+          name: rsync{{i}}::name
+          user: rsync{{i}}::user
+          port: rsync{{i}}::port
   - id: node{{i}}
     from: resources/ro_node
     values: