[envs] Creating a spec from system environment variables

Deployment component had a noce hardcoded to openstack feature - create
a specification based on system environment variables.

This patch ports this feature to Environment manager and extends with
ability to configure not only OpenStack things.

Change-Id: Ic7b09a0a1f6d859f3db92400fb15ed9c1b69ea89

etc/rally.bash_completion

@@ -33,7 +33,7 @@ _rally()
     OPTS["deployment_show"]="--deployment"
     OPTS["deployment_use"]="--deployment"
     OPTS["env_check"]="--env --json --detailed"
-    OPTS["env_create"]="--name --description --extras --spec --json --no-use"
+    OPTS["env_create"]="--name --description --extras --from-sysenv --spec --json --no-use"
     OPTS["env_delete"]="--env --force"
     OPTS["env_destroy"]="--env --skip-cleanup --json --detailed"
     OPTS["env_info"]="--env --json"
@@ -105,4 +105,4 @@ _rally()
     return 0
 }
 
-complete -o filenames -F _rally rally
+complete -o filenames -F _rally rally

rally/cli/commands/env.py

@@ -46,21 +46,42 @@ class EnvCommands(object):
                    help="Env description")
     @cliutils.args("--extras", "-e", type=str, required=False,
                    help="JSON or YAML dict with custom non validate info.")
+    @cliutils.args("--from-sysenv", action="store_true", dest="from_sysenv",
+                   help="Iterate over all available platforms and check system"
+                        " environment for credentials.")
     @cliutils.args("--spec", "-s", type=str, required=False,
                    metavar="<path>", help="Path to env spec.")
     @cliutils.args("--json", action="store_true", dest="to_json",
                    help="Format output as JSON.")
     @cliutils.args("--no-use", action="store_false", dest="do_use",
                    help="Don't set new env as default for future operations.")
-    def create(self, api, name, description=None,
-               extras=None, spec=None, to_json=False, do_use=True):
+    def create(self, api, name, description=None, extras=None,
+               spec=None, from_sysenv=False, to_json=False, do_use=True):
         """Create new environment."""
+
+        if spec is not None and from_sysenv:
+            print("Arguments '--spec' and '--from-sysenv' cannot be used "
+                  "together, use only one of them.")
+            return 1
         spec = spec or {}
         if spec:
             with open(os.path.expanduser(spec), "rb") as f:
                 spec = yaml.safe_load(f.read())
         if extras:
             extras = yaml.safe_load(extras)
+
+        if from_sysenv:
+            result = env_mgr.EnvManager.create_spec_from_sys_environ()
+            spec = result["spec"]
+            _print("Your system environment includes specifications of "
+                   "%s platform(s)." % len(spec), to_json)
+            _print("Discovery information:", to_json)
+            for p_name, p_result in result["discovery_details"].items():
+                _print("\t - %s : %s." % (p_name, p_result["message"]),
+                       to_json)
+
+                if "traceback" in p_result:
+                    _print("".join(p_result["traceback"]), to_json)
         try:
             env = env_mgr.EnvManager.create(
                 name, spec, description=description, extras=extras)

rally/env/env_mgr.py

@@ -13,6 +13,7 @@
 #    under the License.
 
 import copy
+import os
 import traceback
 
 import jsonschema
@@ -357,6 +358,7 @@ class EnvManager(object):
         :param extras: User specified dict with extra options
         :param spec: Specification that contains info about all
                      platform plugins and their arguments.
+        :param config: Reserved for the new feature. Not applicable as for now
         :returns: EnvManager instance corresponding to created Env
         """
         # NOTE(boris-42): this allows to avoid validation copy paste. If spec
@@ -373,6 +375,60 @@ class EnvManager(object):
         self._create_platforms()
         return self
 
+    _FROM_SYS_ENV_FORMAT = {
+        "type": "object",
+        "properties": {
+            "available": {"type": "boolean"},
+            "message": {"type": "string"},
+            "traceback": {"type": "string"},
+            "spec": {"type": "object",
+                     "additionalProperties": True}
+        },
+        "required": ["available"],
+        "additionalProperties": False
+    }
+
+    @classmethod
+    @plugins.ensure_plugins_are_loaded
+    def create_spec_from_sys_environ(cls, description=None, extras=None,
+                                     config=None):
+        """Compose an environment spec based on system environment.
+
+        Iterates over all available platform-representation plugins which
+        checks system environment for credentials.
+
+        :param description: User specified description to include in the spec
+        :param extras: User specified dict with extra options to include in
+            the spec
+        :param config: Reserved for the new feature. Not applicable as for now
+        :returns: A dict with an environment specification and detailed
+            information about discovery
+        """
+
+        details = {}
+        for p in platform.Platform.get_all():
+            try:
+                res = p.create_spec_from_sys_environ(copy.deepcopy(os.environ))
+                jsonschema.validate(res, cls._FROM_SYS_ENV_FORMAT)
+                res.setdefault("message", "Available")
+            except Exception as e:
+                msg = ("Plugin %s.create_from_sys_environ() method is broken"
+                       % p.get_fullname())
+                LOG.exception(msg)
+                res = {"message": msg, "available": False}
+                if not isinstance(e, jsonschema.ValidationError):
+                    res["traceback"] = traceback.format_exc()
+            details[p.get_fullname()] = res
+        spec = dict((k, v.get("spec", {}))
+                    for k, v in details.items() if v["available"])
+        if description is not None:
+            spec["!description"] = description
+        if extras is not None:
+            spec["!extras"] = extras
+        if config is not None:
+            spec["!config"] = config
+        return {"spec": spec, "discovery_details": details}
+
     def rename(self, new_name):
         """Renames env record.
 

rally/env/platform.py

@@ -115,3 +115,9 @@ class Platform(plugin.Plugin, validation.ValidatablePluginMixin):
     def get_validation_context(self):
         """Return a validation context for a platform."""
         return {}
+
+    @classmethod
+    def create_spec_from_sys_environ(cls, sys_environ):
+        """Check system env for credentials and return a spec if present."""
+        return {"available": False,
+                "message": "Skipped. No credentials found."}

rally/plugins/openstack/platforms/existing.py

@@ -195,3 +195,58 @@ class OpenStack(platform.Platform):
 
     def _get_validation_context(self):
         return {"users@openstack": {}}
+
+    @classmethod
+    def create_spec_from_sys_environ(cls, sys_environ):
+
+        from oslo_utils import strutils
+
+        required_env_vars = ["OS_AUTH_URL", "OS_USERNAME", "OS_PASSWORD"]
+        missing_env_vars = [v for v in required_env_vars if
+                            v not in sys_environ]
+        if missing_env_vars:
+            return {"available": False,
+                    "message": "The following variable(s) are missed: %s" %
+                               missing_env_vars}
+        tenant_name = sys_environ.get("OS_PROJECT_NAME",
+                                      sys_environ.get("OS_TENANT_NAME"))
+        if tenant_name is None:
+            return {"available": False,
+                    "message": "One of OS_PROJECT_NAME or OS_TENANT_NAME "
+                               "should be specified."}
+
+        endpoint_type = sys_environ.get("OS_ENDPOINT_TYPE",
+                                        sys_environ.get("OS_INTERFACE"))
+        if endpoint_type and "URL" in endpoint_type:
+            endpoint_type = endpoint_type.replace("URL", "")
+
+        spec = {
+            "auth_url": sys_environ["OS_AUTH_URL"],
+            "admin": {
+                "username": sys_environ["OS_USERNAME"],
+                "password": sys_environ["OS_PASSWORD"],
+                "tenant_name": tenant_name
+            },
+            "endpoint_type": endpoint_type,
+            "region_name": sys_environ.get("OS_REGION_NAME", ""),
+            "https_cacert": sys_environ.get("OS_CACERT", ""),
+            "https_insecure": strutils.bool_from_string(
+                sys_environ.get("OS_INSECURE")),
+            "profiler_hmac_key": sys_environ.get("OSPROFILER_HMAC_KEY"),
+            "profiler_conn_str": sys_environ.get("OSPROFILER_CONN_STR")
+        }
+
+        user_domain_name = sys_environ.get("OS_USER_DOMAIN_NAME")
+        project_domain_name = sys_environ.get("OS_PROJECT_DOMAIN_NAME")
+        identity_api_version = sys_environ.get(
+            "OS_IDENTITY_API_VERSION", sys_environ.get("IDENTITY_API_VERSION"))
+        if (identity_api_version == "3" or
+                (identity_api_version is None and
+                 (user_domain_name or project_domain_name))):
+            # it is Keystone v3 and it has another config scheme
+            spec["admin"]["project_name"] = spec["admin"].pop("tenant_name")
+            spec["admin"]["user_domain_name"] = user_domain_name or "Default"
+            project_domain_name = project_domain_name or "Default"
+            spec["admin"]["project_domain_name"] = project_domain_name
+
+        return {"spec": spec, "available": True, "message": "Available"}

tests/unit/cli/commands/test_env.py

@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import collections
 import datetime as dt
 import json
 import uuid
@@ -97,6 +98,50 @@ class EnvCommandsTestCase(test.TestCase):
             mock.call(mock.ANY)
         ])
 
+    @mock.patch("rally.cli.commands.env.EnvCommands._show")
+    @mock.patch("rally.env.env_mgr.EnvManager.create_spec_from_sys_environ")
+    @mock.patch("rally.env.env_mgr.EnvManager.create")
+    @mock.patch("rally.cli.commands.env.open", create=True)
+    @mock.patch("rally.cli.commands.env.print")
+    def test_create_from_sys_env(
+            self, mock_print, mock_open, mock_env_manager_create,
+            mock_env_manager_create_spec_from_sys_environ,
+            mock_env_commands__show):
+        result = {
+            "spec": {"foo": mock.Mock()},
+            "discovery_details": collections.OrderedDict([
+                ("foo", {"available": True, "message": "available"}),
+                ("bar", {"available": False, "message": "not available",
+                         "traceback": "trace"})
+            ])
+        }
+        mock_env_manager_create_spec_from_sys_environ.return_value = result
+
+        self.assertEqual(
+            0, self.env.create(self.api, "n", "d", spec=None,
+                               from_sysenv=True, do_use=False))
+        self.assertEqual(
+            [
+                # check that the number of listed platforms is right
+                mock.call("Your system environment includes specifications of"
+                          " 1 platform(s)."),
+                mock.call("Discovery information:"),
+                mock.call("\t - foo : available."),
+                mock.call("\t - bar : not available."),
+                mock.call("trace")
+            ], mock_print.call_args_list)
+
+        mock_env_manager_create_spec_from_sys_environ.assert_called_once_with()
+        mock_env_manager_create.assert_called_once_with(
+            "n", result["spec"], description="d", extras=None)
+        self.assertFalse(mock_open.called)
+
+    @mock.patch("rally.cli.commands.env.print")
+    def test_create_with_incompatible_arguments(self, mock_print):
+        self.assertEqual(
+            1, self.env.create(self.api, "n", "d", spec="asd",
+                               from_sysenv=True))
+
     @mock.patch("rally.env.env_mgr.EnvManager.create")
     @mock.patch("rally.cli.commands.env.print")
     def test_create_exception(self, mock_print, mock_env_manager_create):

tests/unit/env/test_env_mgr.py

@@ -14,6 +14,7 @@
 
 import copy
 import datetime as dt
+import os
 
 import mock
 
@@ -354,6 +355,83 @@ class EnvManagerTestCase(test.TestCase):
             }
         )
 
+    @mock.patch.dict(os.environ, values={"KEY": "value"}, clear=True)
+    @mock.patch("rally.env.platform.Platform.get_all")
+    def test_create_spec_from_sys_environ(self, mock_platform_get_all):
+
+        # let's cover all positive and minor cases at once
+
+        class Foo1Platform(platform.Platform):
+            """This platform doesn't override original methods"""
+
+            @classmethod
+            def get_fullname(cls):
+                return cls.__name__
+
+        class Foo2Platform(Foo1Platform):
+            """This platform should try to modify sys environment"""
+
+            @classmethod
+            def create_spec_from_sys_environ(cls, sys_environ):
+                for key in list(sys_environ.keys()):
+                    sys_environ.pop(key)
+                return platform.Platform.create_spec_from_sys_environ({})
+
+        class Foo3Platform(Foo1Platform):
+            """This platform rely on one sys argument."""
+
+            @classmethod
+            def create_spec_from_sys_environ(cls, sys_environ):
+                self.assertIn("KEY", sys_environ)
+                return {"spec": {"KEY": sys_environ["KEY"]},
+                        "available": True}
+
+        class Foo4Platform(Foo1Platform):
+            """This platform raises an error!"""
+
+            @classmethod
+            def create_spec_from_sys_environ(cls, sys_environ):
+                raise KeyError("Ooopes")
+
+        class Foo5Platform(Foo1Platform):
+            """This platform returns invalid data."""
+
+            @classmethod
+            def create_spec_from_sys_environ(cls, sys_environ):
+                return {"foo": "bar"}
+
+        mock_platform_get_all.return_value = [
+            Foo1Platform, Foo2Platform, Foo3Platform, Foo4Platform,
+            Foo5Platform]
+
+        result = env_mgr.EnvManager.create_spec_from_sys_environ()
+        self.assertEqual({"Foo3Platform": {"KEY": "value"}}, result["spec"])
+        self.assertEqual(
+            {"Foo1Platform", "Foo2Platform", "Foo3Platform", "Foo4Platform",
+             "Foo5Platform"},
+            set(result["discovery_details"].keys()))
+        result = result["discovery_details"]
+        default_msg = "Skipped. No credentials found."
+
+        self.assertFalse(result["Foo1Platform"]["available"])
+        self.assertEqual(default_msg, result["Foo1Platform"]["message"])
+
+        self.assertFalse(result["Foo2Platform"]["available"])
+        self.assertEqual(default_msg, result["Foo2Platform"]["message"])
+
+        self.assertTrue(result["Foo3Platform"]["available"])
+        self.assertEqual("Available", result["Foo3Platform"]["message"])
+
+        self.assertFalse(result["Foo4Platform"]["available"])
+        self.assertIn("method is broken", result["Foo4Platform"]["message"])
+        self.assertIn("traceback", result["Foo4Platform"])
+
+        self.assertFalse(result["Foo5Platform"]["available"])
+        self.assertIn("method is broken", result["Foo5Platform"]["message"])
+        self.assertNotIn("traceback", result["Foo5Platform"])
+
+        mock_platform_get_all.assert_called_once_with()
+
     @mock.patch("rally.common.db.env_rename")
     def test_rename(self, mock_env_rename):
         env = env_mgr.EnvManager({"uuid": "11", "name": "n"})

tests/unit/plugins/openstack/platforms/test_existing.py

@@ -127,6 +127,77 @@ class ExistingPlatformTestCase(test_platform.PlatformBaseTestCase):
             ),
             existing.OpenStack(spec).create())
 
+    def test_create_spec_from_sys_environ(self):
+        # keystone v2
+        sys_env = {
+            "OS_AUTH_URL": "https://example.com",
+            "OS_USERNAME": "user",
+            "OS_PASSWORD": "pass",
+            "OS_TENANT_NAME": "projectX",
+            "OS_INTERFACE": "publicURL",
+            "OS_REGION_NAME": "Region1",
+            "OS_CACERT": "Cacert",
+            "OS_INSECURE": True,
+            "OSPROFILER_HMAC_KEY": "key",
+            "OSPROFILER_CONN_STR": "https://example2.com",
+        }
+
+        result = existing.OpenStack.create_spec_from_sys_environ(sys_env)
+        self.assertTrue(result["available"])
+        self.assertEqual(
+            {
+                "admin": {
+                    "username": "user",
+                    "tenant_name": "projectX",
+                    "password": "pass"
+                },
+                "auth_url": "https://example.com",
+                "endpoint_type": "public",
+                "region_name": "Region1",
+                "https_cacert": "Cacert",
+                "https_insecure": True,
+                "profiler_hmac_key": "key",
+                "profiler_conn_str": "https://example2.com"
+            }, result["spec"])
+
+        # keystone v3
+        sys_env["OS_IDENTITY_API_VERSION"] = "3"
+
+        result = existing.OpenStack.create_spec_from_sys_environ(sys_env)
+        print(json.dumps(result["spec"], indent=4))
+        self.assertEqual(
+            {
+                "admin": {
+                    "username": "user",
+                    "project_name": "projectX",
+                    "user_domain_name": "Default",
+                    "password": "pass",
+                    "project_domain_name": "Default"
+                },
+                "endpoint_type": "public",
+                "auth_url": "https://example.com",
+                "region_name": "Region1",
+                "https_cacert": "Cacert",
+                "https_insecure": True,
+                "profiler_hmac_key": "key",
+                "profiler_conn_str": "https://example2.com"
+            }, result["spec"])
+
+    def test_create_spec_from_sys_environ_fails_with_missing_vars(self):
+        sys_env = {"OS_AUTH_URL": "https://example.com"}
+        result = existing.OpenStack.create_spec_from_sys_environ(sys_env)
+        self.assertFalse(result["available"])
+        self.assertIn("OS_USERNAME", result["message"])
+        self.assertIn("OS_PASSWORD", result["message"])
+        self.assertNotIn("OS_AUTH_URL", result["message"])
+
+        sys_env = {"OS_AUTH_URL": "https://example.com",
+                   "OS_USERNAME": "user",
+                   "OS_PASSWORD": "pass"}
+        result = existing.OpenStack.create_spec_from_sys_environ(sys_env)
+        self.assertFalse(result["available"])
+        self.assertIn("OS_PROJECT_NAME or OS_TENANT_NAME", result["message"])
+
     def test_destroy(self):
         self.assertIsNone(existing.OpenStack({}).destroy())