Merge "Add job-types endpoint"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
af2f0e9a29
|
|
@ -729,21 +729,21 @@ A Job may be run on an existing cluster or a new transient cluster may be create
|
|||
|
||||
**Job ops**
|
||||
|
||||
+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+
|
||||
| Verb | URI | Description |
|
||||
+=================+===================================================================+=====================================================+
|
||||
| GET | /v1.1/{tenant_id}/jobs | Lists all created Jobs |
|
||||
+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+
|
||||
| GET | /v1.1/{tenant_id}/jobs/<job_id> | Shows info about specified Job by id |
|
||||
+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+
|
||||
| POST | /v1.1/{tenant_id}/jobs | Create a new Job object |
|
||||
+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+
|
||||
| DELETE | /v1.1/{tenant_id}/jobs/<job_id> | Removes specified Job |
|
||||
+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+
|
||||
| GET | /v1.1/{tenant_id}/jobs/config-hints/<job_type> | Shows default configuration by specified Job type |
|
||||
+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+
|
||||
| POST | /v1.1/{tenant_id}/jobs/<job_id>/execute | Starts Job executing |
|
||||
+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+
|
||||
+-----------------+-------------------------------------------------------------------+--------------------------------------------------------+
|
||||
| Verb | URI | Description |
|
||||
+=================+===================================================================+========================================================+
|
||||
| GET | /v1.1/{tenant_id}/jobs | Lists all created Jobs |
|
||||
+-----------------+-------------------------------------------------------------------+--------------------------------------------------------+
|
||||
| GET | /v1.1/{tenant_id}/jobs/<job_id> | Shows info about specified Job by id |
|
||||
+-----------------+-------------------------------------------------------------------+--------------------------------------------------------+
|
||||
| POST | /v1.1/{tenant_id}/jobs | Create a new Job object |
|
||||
+-----------------+-------------------------------------------------------------------+--------------------------------------------------------+
|
||||
| DELETE | /v1.1/{tenant_id}/jobs/<job_id> | Removes specified Job |
|
||||
+-----------------+-------------------------------------------------------------------+--------------------------------------------------------+
|
||||
| GET | /v1.1/{tenant_id}/jobs/config-hints/<job_type> | Shows default configuration for Job type (deprecated) |
|
||||
+-----------------+-------------------------------------------------------------------+--------------------------------------------------------+
|
||||
| POST | /v1.1/{tenant_id}/jobs/<job_id>/execute | Starts Job executing |
|
||||
+-----------------+-------------------------------------------------------------------+--------------------------------------------------------+
|
||||
|
||||
**Examples**
|
||||
|
||||
|
|
@ -996,6 +996,8 @@ Errors: none
|
|||
|
||||
This operation returns hints for configuration parameters which can be applied during job execution.
|
||||
|
||||
(deprecated) For config-hints, the *job-types* endpoint should be used instead.
|
||||
|
||||
This operation does not require a request body.
|
||||
|
||||
**Note**
|
||||
|
|
@ -1587,3 +1589,420 @@ The following json response represents a Job Execution object returned from Saha
|
|||
"job_configs": {},
|
||||
"created_at": "2013-10-17 13:51:11.671977"
|
||||
}
|
||||
|
||||
|
||||
7 Job Types
|
||||
===========
|
||||
|
||||
**Description**
|
||||
|
||||
Each plugin that supports EDP will support specific job types.
|
||||
Different versions of a plugin may actually support different job types.
|
||||
Configuration options will vary by plugin, version, and job type.
|
||||
|
||||
This endpoint provides information on which job types are supported by
|
||||
which plugins and optionally how they may be configured.
|
||||
|
||||
**Job Binary Internal ops**
|
||||
|
||||
+-----------------+------------------------------------------------------+----------------------------------------------------------+
|
||||
| Verb | URI | Description |
|
||||
+=================+======================================================+==========================================================+
|
||||
| GET | /v1.1/{tenant_id}/job-types | Lists job types supported by all versions of all plugins |
|
||||
+-----------------+------------------------------------------------------+----------------------------------------------------------+
|
||||
| GET | /v1.1/{tenant_id}/job-types?plugin=<plugin_name> | Filter results by plugin name |
|
||||
+-----------------+------------------------------------------------------+----------------------------------------------------------+
|
||||
| GET | /v1.1/{tenant_id}/job-types?version=<plugin_version> | Filter results by plugin version |
|
||||
+-----------------+------------------------------------------------------+----------------------------------------------------------+
|
||||
| GET | /v1.1/{tenant_id}/job-types?type=<job_type> | Filter results by job type |
|
||||
+-----------------+------------------------------------------------------+----------------------------------------------------------+
|
||||
| GET | /v1.1/{tenant_id}/job-types?hints=true | Include configuration hints in results |
|
||||
+-----------------+------------------------------------------------------+----------------------------------------------------------+
|
||||
|
||||
Note that multiple search filters may be combined in the query string with *&* (for example ?type=Java&type=Pig&plugin=vanilla).
|
||||
|
||||
**Examples**
|
||||
|
||||
7.1 List all Job Types
|
||||
----------------------
|
||||
|
||||
.. http:get:: /v1.1/{tenant_id}/job-types
|
||||
|
||||
Normal Response Code: 200 (OK)
|
||||
|
||||
Errors: none
|
||||
|
||||
For all job types show which versions of which plugins support them
|
||||
|
||||
This operation does not require a request body.
|
||||
|
||||
**Example**:
|
||||
**request**
|
||||
|
||||
.. sourcecode:: http
|
||||
|
||||
GET http://sahara:8386/v1.1/775181/job-types
|
||||
|
||||
**response**
|
||||
|
||||
.. sourcecode:: http
|
||||
|
||||
HTTP/1.1 200 OK
|
||||
Content-Type: application/json
|
||||
|
||||
.. sourcecode:: json
|
||||
|
||||
{
|
||||
"job_types": [
|
||||
{
|
||||
"name": "Hive",
|
||||
"plugins": [
|
||||
{
|
||||
"description": "The Apache Vanilla plugin.",
|
||||
"name": "vanilla",
|
||||
"title": "Vanilla Apache Hadoop",
|
||||
"versions": {
|
||||
"1.2.1": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "The Hortonworks Sahara plugin.",
|
||||
"name": "hdp",
|
||||
"title": "Hortonworks Data Platform",
|
||||
"versions": {
|
||||
"1.3.2": {},
|
||||
"2.0.6": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Java",
|
||||
"plugins": [
|
||||
{
|
||||
"description": "The Apache Vanilla plugin.",
|
||||
"name": "vanilla",
|
||||
"title": "Vanilla Apache Hadoop",
|
||||
"versions": {
|
||||
"1.2.1": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "The Hortonworks Sahara plugin.",
|
||||
"name": "hdp",
|
||||
"title": "Hortonworks Data Platform",
|
||||
"versions": {
|
||||
"1.3.2": {},
|
||||
"2.0.6": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "MapReduce",
|
||||
"plugins": [
|
||||
{
|
||||
"description": "The Apache Vanilla plugin.",
|
||||
"name": "vanilla",
|
||||
"title": "Vanilla Apache Hadoop",
|
||||
"versions": {
|
||||
"1.2.1": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "The Hortonworks Sahara plugin.",
|
||||
"name": "hdp",
|
||||
"title": "Hortonworks Data Platform",
|
||||
"versions": {
|
||||
"1.3.2": {},
|
||||
"2.0.6": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "MapReduce.Streaming",
|
||||
"plugins": [
|
||||
{
|
||||
"description": "The Apache Vanilla plugin.",
|
||||
"name": "vanilla",
|
||||
"title": "Vanilla Apache Hadoop",
|
||||
"versions": {
|
||||
"1.2.1": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "The Hortonworks Sahara plugin.",
|
||||
"name": "hdp",
|
||||
"title": "Hortonworks Data Platform",
|
||||
"versions": {
|
||||
"1.3.2": {},
|
||||
"2.0.6": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pig",
|
||||
"plugins": [
|
||||
{
|
||||
"description": "The Apache Vanilla plugin.",
|
||||
"name": "vanilla",
|
||||
"title": "Vanilla Apache Hadoop",
|
||||
"versions": {
|
||||
"1.2.1": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "The Hortonworks Sahara plugin.",
|
||||
"name": "hdp",
|
||||
"title": "Hortonworks Data Platform",
|
||||
"versions": {
|
||||
"1.3.2": {},
|
||||
"2.0.6": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
7.2 List a subset of Job Types
|
||||
------------------------------
|
||||
|
||||
.. http:get:: /v1.1/{tenant_id}/job-types?type=<job_type>&type=<job_type>
|
||||
|
||||
Normal Response Code: 200 (OK)
|
||||
|
||||
Errors: none
|
||||
|
||||
For the specified job types show which versions of which plugins support them
|
||||
|
||||
This operation does not require a request body.
|
||||
|
||||
**Example**:
|
||||
**request**
|
||||
|
||||
.. sourcecode:: http
|
||||
|
||||
GET http://sahara:8386/v1.1/775181/job-types?type=Hive&type=Java
|
||||
|
||||
**response**
|
||||
|
||||
.. sourcecode:: http
|
||||
|
||||
HTTP/1.1 200 OK
|
||||
Content-Type: application/json
|
||||
|
||||
.. sourcecode:: json
|
||||
|
||||
{
|
||||
"job_types": [
|
||||
{
|
||||
"name": "Hive",
|
||||
"plugins": [
|
||||
{
|
||||
"description": "The Apache Vanilla plugin.",
|
||||
"name": "vanilla",
|
||||
"title": "Vanilla Apache Hadoop",
|
||||
"versions": {
|
||||
"1.2.1": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "The Hortonworks Sahara plugin.",
|
||||
"name": "hdp",
|
||||
"title": "Hortonworks Data Platform",
|
||||
"versions": {
|
||||
"1.3.2": {},
|
||||
"2.0.6": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Java",
|
||||
"plugins": [
|
||||
{
|
||||
"description": "The Apache Vanilla plugin.",
|
||||
"name": "vanilla",
|
||||
"title": "Vanilla Apache Hadoop",
|
||||
"versions": {
|
||||
"1.2.1": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "The Hortonworks Sahara plugin.",
|
||||
"name": "hdp",
|
||||
"title": "Hortonworks Data Platform",
|
||||
"versions": {
|
||||
"1.3.2": {},
|
||||
"2.0.6": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
7.3 List all Job Types supported by a plugin version
|
||||
----------------------------------------------------
|
||||
|
||||
.. http:get:: /v1.1/{tenant_id}/job-types?plugin=<plugin_name>&version=<plugin_version>
|
||||
|
||||
Normal Response Code: 200 (OK)
|
||||
|
||||
Errors: none
|
||||
|
||||
Show all of the job types that are supported by a specified version
|
||||
of a specified plugin
|
||||
|
||||
This operation does not require a request body.
|
||||
|
||||
**Example**:
|
||||
**request**
|
||||
|
||||
.. sourcecode:: http
|
||||
|
||||
GET http://sahara:8386/v1.1/775181/job-types?plugin=hdp&version=2.0.6
|
||||
|
||||
**response**
|
||||
|
||||
.. sourcecode:: http
|
||||
|
||||
HTTP/1.1 200 OK
|
||||
Content-Type: application/json
|
||||
|
||||
.. sourcecode:: json
|
||||
|
||||
{
|
||||
"job_types": [
|
||||
{
|
||||
"name": "Hive",
|
||||
"plugins": [
|
||||
{
|
||||
"description": "The Hortonworks Sahara plugin.",
|
||||
"name": "hdp",
|
||||
"title": "Hortonworks Data Platform",
|
||||
"versions": {
|
||||
"2.0.6": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Java",
|
||||
"plugins": [
|
||||
{
|
||||
"description": "The Hortonworks Sahara plugin.",
|
||||
"name": "hdp",
|
||||
"title": "Hortonworks Data Platform",
|
||||
"versions": {
|
||||
"2.0.6": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "MapReduce",
|
||||
"plugins": [
|
||||
{
|
||||
"description": "The Hortonworks Sahara plugin.",
|
||||
"name": "hdp",
|
||||
"title": "Hortonworks Data Platform",
|
||||
"versions": {
|
||||
"2.0.6": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "MapReduce.Streaming",
|
||||
"plugins": [
|
||||
{
|
||||
"description": "The Hortonworks Sahara plugin.",
|
||||
"name": "hdp",
|
||||
"title": "Hortonworks Data Platform",
|
||||
"versions": {
|
||||
"2.0.6": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pig",
|
||||
"plugins": [
|
||||
{
|
||||
"description": "The Hortonworks Sahara plugin.",
|
||||
"name": "hdp",
|
||||
"title": "Hortonworks Data Platform",
|
||||
"versions": {
|
||||
"2.0.6": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
7.4 Show configuration hints for a specific Job Type supported by a specific plugin version
|
||||
-------------------------------------------------------------------------------------------
|
||||
|
||||
.. http:get:: /v1.1/{tenant_id}/job-types?hints=true&plugin=<plugin_name>&version=<plugin_version>&type=<job_type>
|
||||
|
||||
Normal Response Code: 200 (OK)
|
||||
|
||||
Errors: none
|
||||
|
||||
Show the configuration hints for a single job type supported by a particular plugin version
|
||||
|
||||
This operation does not require a request body.
|
||||
|
||||
**Example**
|
||||
**request**
|
||||
|
||||
.. sourcecode:: http
|
||||
|
||||
GET http://sahara/v1.1/775181/job-types?hints=true&plugin=hdp&version=1.3.2&type=Hive
|
||||
|
||||
**response**
|
||||
|
||||
.. sourcecode:: http
|
||||
|
||||
HTTP/1.1 200 OK
|
||||
Content-Type: application/json
|
||||
|
||||
.. sourcecode:: json
|
||||
|
||||
{
|
||||
"job_types": [
|
||||
{
|
||||
"name": "Hive",
|
||||
"plugins": [
|
||||
{
|
||||
"description": "The Hortonworks Sahara plugin.",
|
||||
"name": "hdp",
|
||||
"title": "Hortonworks Data Platform",
|
||||
"versions": {
|
||||
"1.3.2": {
|
||||
"job_config": {
|
||||
"args": {},
|
||||
"configs": [
|
||||
{
|
||||
"description": "Reduce tasks.",
|
||||
"name": "mapred.reduce.tasks",
|
||||
"value": "-1"
|
||||
}
|
||||
],
|
||||
"params": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
This is an abbreviated example that shows imaginary config hints.
|
||||
|
|
|
|||
|
|
@ -60,5 +60,7 @@
|
|||
"job-binary-internals:create": "",
|
||||
"job-binary-internals:get": "",
|
||||
"job-binary-internals:delete": "",
|
||||
"job-binary-internals:get_data": ""
|
||||
"job-binary-internals:get_data": "",
|
||||
|
||||
"job-types:get_all": ""
|
||||
}
|
||||
|
|
|
|||
|
|
@ -151,8 +151,18 @@ def job_config_hints_get(job_type):
|
|||
return u.render(api.get_job_config_hints(job_type))
|
||||
|
||||
|
||||
@rest.get('/job-types')
|
||||
@acl.enforce("job-types:get_all")
|
||||
def job_types_get():
|
||||
# We want to use flat=False with to_dict() so that
|
||||
# the value of each arg is given as a list. This supports
|
||||
# filters of the form ?type=Pig&type=Java, etc.
|
||||
return u.render(job_types=api.get_job_types(
|
||||
**u.get_request_args().to_dict(flat=False)))
|
||||
|
||||
# Job binary ops
|
||||
|
||||
|
||||
@rest.post('/job-binaries')
|
||||
@acl.enforce("job-binaries:create")
|
||||
@v.validate(v_j_b.JOB_BINARY_SCHEMA, v_j_b.check_job_binary)
|
||||
|
|
|
|||
|
|
@ -65,6 +65,14 @@ class ProvisioningPluginBase(plugins_base.PluginInterface):
|
|||
def get_edp_engine(self, cluster, job_type):
|
||||
pass
|
||||
|
||||
@plugins_base.optional
|
||||
def get_edp_job_types(self, versions=[]):
|
||||
return {}
|
||||
|
||||
@plugins_base.optional
|
||||
def get_edp_config_hints(self, job_type, version):
|
||||
return {}
|
||||
|
||||
@plugins_base.required_with_default
|
||||
def get_open_ports(self, node_group):
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -15,11 +15,14 @@
|
|||
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
import six
|
||||
|
||||
from sahara import conductor as c
|
||||
from sahara import context
|
||||
from sahara import exceptions as ex
|
||||
from sahara.i18n import _LE
|
||||
from sahara.plugins import base as plugin_base
|
||||
from sahara.plugins import provisioning
|
||||
from sahara.service.edp.binary_retrievers import dispatch
|
||||
from sahara.service.edp import job_manager as manager
|
||||
from sahara.utils import edp
|
||||
|
|
@ -40,6 +43,66 @@ def setup_edp_api(ops):
|
|||
OPS = ops
|
||||
|
||||
|
||||
def get_job_types(**kwargs):
|
||||
# Return a dictionary of all the job types that can be run
|
||||
# by this instance of Sahara. For each job type, the value
|
||||
# will be a list of plugins that support the job type. For
|
||||
# each plugin, include a dictionary of the versions that
|
||||
# support the job type.
|
||||
|
||||
# All entries in kwargs are expected to have list values
|
||||
hints = kwargs.get("hints", ["false"])[0].lower() == "true"
|
||||
|
||||
plugin_names = kwargs.get("plugin", [])
|
||||
all_plugins = plugin_base.PLUGINS.get_plugins(
|
||||
base=provisioning.ProvisioningPluginBase)
|
||||
if plugin_names:
|
||||
plugins = filter(lambda x: x.name in plugin_names, all_plugins)
|
||||
else:
|
||||
plugins = all_plugins
|
||||
|
||||
job_types = kwargs.get("type", edp.JOB_TYPES_ALL)
|
||||
versions = kwargs.get("version", [])
|
||||
|
||||
res = []
|
||||
|
||||
for job_type in job_types:
|
||||
# All job types supported by all versions of the plugin.
|
||||
# This is a dictionary where keys are plugin version
|
||||
# strings and values are lists of job types
|
||||
job_entry = {"name": job_type,
|
||||
"plugins": []}
|
||||
|
||||
for plugin in plugins:
|
||||
types_for_plugin = plugin.get_edp_job_types(versions)
|
||||
|
||||
# dict returns a new object so we are not modifying the plugin
|
||||
p = plugin.dict
|
||||
|
||||
# Find only the versions of this plugin that support the job.
|
||||
# Additionally, instead of a list we want a dictionary of
|
||||
# plugin versions with corresponding config hints
|
||||
p["versions"] = {}
|
||||
|
||||
for version, supported_types in six.iteritems(types_for_plugin):
|
||||
if job_type in supported_types:
|
||||
if hints:
|
||||
config_hints = plugin.get_edp_config_hints(job_type,
|
||||
version)
|
||||
else:
|
||||
config_hints = {}
|
||||
p["versions"][version] = config_hints
|
||||
|
||||
# If we found at least one version of the plugin that
|
||||
# supports the job type, add the plugin to the result
|
||||
if p["versions"]:
|
||||
job_entry["plugins"].append(p)
|
||||
|
||||
if job_entry["plugins"]:
|
||||
res.append(job_entry)
|
||||
return res
|
||||
|
||||
|
||||
def get_job_config_hints(job_type):
|
||||
return manager.get_job_config_hints(job_type)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue