openstack
/
sahara-dashboard
Code Issues Proposed changes

Merge "Support of S3 data sources in dashboard"

This commit is contained in:
Zuul 2018-07-26 08:28:41 +00:00 committed by Gerrit Code Review
parent 95aca19615 43c1c36c0d
commit bc2d21d1d0
5 changed files with 144 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
sahara_dashboard/api/sahara.py
View File

@ -446,16 +446,18 @@ def cluster_update_acl_rules(request, cl_id, is_public=None,
def data_source_create(request, name, description, ds_type, url,
credential_user=None, credential_pass=None,
is_public=None, is_protected=None):
is_public=None, is_protected=None,
s3_credentials=None):
return client(request).data_sources.create(
name=name,
description=description,
data_source_type=ds_type,
url=url,
credential_user=credential_user,
credential_pass=credential_pass,
credential_user=credential_user or None,
credential_pass=credential_pass or None,
is_public=is_public,
is_protected=is_protected)
is_protected=is_protected,
s3_credentials=s3_credentials)
def data_source_list(request, search_opts=None, limit=None, marker=None):

35
sahara_dashboard/content/data_processing/jobs/data_sources/tests.py
View File

@ -92,10 +92,11 @@ class DataProcessingDataSourceTests(test.TestCase):
data_source.description,
data_source.type,
data_source.url,
"",
"",
'',
'',
is_public=False,
is_protected=False)
is_protected=False,
s3_credentials=None)
self.assertNoFormErrors(res)
self.assertRedirectsNoFollow(res, INDEX_URL)
@ -164,6 +165,32 @@ class DataProcessingDataSourceTests(test.TestCase):
IsA(six.text_type),
IsA(six.text_type),
IsA(str),
"", "", is_public=False, is_protected=False)
'', '', is_public=False, is_protected=False, s3_credentials=None)
self.mock_share_list.assert_called_once_with(IsHttpRequest())
self.assertNoFormErrors(res)
@test.create_mocks({api.sahara: ('data_source_create',)})
def test_create_s3(self):
form_data = {
"data_source_type": "s3",
"data_source_url": "s3a://a/b",
"data_source_credential_accesskey": "acc",
"data_source_credential_secretkey": "sec",
"data_source_credential_endpoint": "pointy.end",
"data_source_credential_s3_bucket_in_url": False,
"data_source_credential_s3_ssl": True,
"data_source_name": "tests3",
"data_source_description": "Test s3 description"
}
res = self.client.post(CREATE_URL, form_data)
self.mock_data_source_create.return_value = True
self.mock_data_source_create.assert_called_once_with(
IsHttpRequest(),
IsA(six.text_type),
IsA(six.text_type),
IsA(six.text_type),
IsA(six.text_type),
IsA(six.text_type),
IsA(six.text_type),
is_public=False, is_protected=False, s3_credentials=IsA(dict))
self.assertNoFormErrors(res)

75
sahara_dashboard/content/data_processing/jobs/data_sources/workflows/create.py
View File

@ -52,7 +52,8 @@ class GeneralConfigAction(workflows.Action):
"data-ds_type-manila": _("Path on share"),
"data-ds_type-swift": _("URL"),
"data-ds_type-hdfs": _("URL"),
"data-ds_type-maprfs": _("URL")
"data-ds_type-maprfs": _("URL"),
"data-ds_type-s3": _("URL"),
}))
data_source_credential_user = forms.CharField(
@ -74,6 +75,56 @@ class GeneralConfigAction(workflows.Action):
label=_("Source password"),
required=False)
data_source_credential_accesskey = forms.CharField(
label=_("S3 accsss key"),
required=False,
widget=forms.TextInput(attrs={
"class": "switched",
"data-switch-on": "ds_type",
"data-ds_type-s3": _("S3 access key")
}))
data_source_credential_secretkey = forms.CharField(
widget=forms.PasswordInput(attrs={
'class': 'switched',
'data-switch-on': 'ds_type',
'data-ds_type-s3': _("S3 secret key"),
'autocomplete': 'off'
}),
label=_("S3 secret key"),
required=False)
data_source_credential_endpoint = forms.CharField(
label=_("S3 endpoint"),
required=False,
help_text=_("Endpoint should be specified without protocol"),
widget=forms.TextInput(attrs={
"class": "switched",
"data-switch-on": "ds_type",
"data-ds_type-s3": _("S3 endpoint")
}))
data_source_credential_s3_ssl = forms.BooleanField(
label=_("Use SSL"),
required=False,
initial=False,
widget=forms.CheckboxInput(attrs={
"class": "switched",
"data-switch-on": "ds_type",
"data-ds_type-s3": _("Use SSL")
}))
data_source_credential_s3_bucket_in_path = forms.BooleanField(
label=_("Use bucket in path"),
required=False,
initial=True,
help_text=_("If checked, bucket will be in path instead of host"),
widget=forms.CheckboxInput(attrs={
"class": "switched",
"data-switch-on": "ds_type",
"data-ds_type-s3": _("Use bucket in path")
}))
data_source_description = forms.CharField(
label=_("Description"),
required=False,
@ -87,7 +138,8 @@ class GeneralConfigAction(workflows.Action):
self.fields["data_source_type"].choices = [("swift", "Swift"),
("hdfs", "HDFS"),
("maprfs", "MapR FS")]
("maprfs", "MapR FS"),
("s3", "S3")]
# If Manila is running, enable it as a choice for a data source
if saharaclient.base.is_service_enabled(request, 'share'):
self.fields["data_source_type"].choices.append(
@ -142,6 +194,22 @@ class CreateDataSource(workflows.Workflow):
default_steps = (GeneralConfig, )
def handle(self, request, context):
s3_credentials = {}
if context["general_data_source_type"] == "s3":
if context.get("general_data_source_credential_accesskey", None):
s3_credentials["accesskey"] = context[
"general_data_source_credential_accesskey"]
if context.get("general_data_source_credential_secretkey", None):
s3_credentials["secretkey"] = context[
"general_data_source_credential_secretkey"]
if context.get("general_data_source_credential_endpoint", None):
s3_credentials["endpoint"] = context[
"general_data_source_credential_endpoint"]
s3_credentials["bucket_in_path"] = context[
"general_data_source_credential_s3_bucket_in_path"]
s3_credentials["ssl"] = context[
"general_data_source_credential_s3_ssl"]
s3_credentials = s3_credentials or None
try:
self.object = saharaclient.data_source_create(
request,
@ -152,7 +220,8 @@ class CreateDataSource(workflows.Workflow):
context.get("general_data_source_credential_user", None),
context.get("general_data_source_credential_pass", None),
is_public=context['general_is_public'],
is_protected=context['general_is_protected']
is_protected=context['general_is_protected'],
s3_credentials=s3_credentials
)
hlps = helpers.Helpers(request)

36
sahara_dashboard/content/data_processing/jobs/data_sources/workflows/edit.py
View File

@ -37,6 +37,11 @@ class EditDataSource(create.CreateDataSource):
"data_source_url": "url",
"data_source_credential_user": None,
"data_source_credential_pass": None,
"data_source_credential_accesskey": None,
"data_source_credential_secretkey": None,
"data_source_credential_endpoint": None,
"data_source_credential_s3_ssl": None,
"data_source_credential_s3_bucket_in_path": None,
"data_source_manila_share": None,
'is_public': "is_public",
'is_protected': "is_protected"
@ -72,17 +77,36 @@ class EditDataSource(create.CreateDataSource):
def handle(self, request, context):
try:
credentials = {}
if context["general_data_source_type"] == "swift":
credentials["user"] = context.get(
"general_data_source_credential_user", None)
credentials["password"] = context.get(
"general_data_source_credential_pass", None)
elif context["general_data_source_type"] == "s3":
if context.get("general_data_source_credential_accesskey",
None):
credentials["accesskey"] = context[
"general_data_source_credential_accesskey"]
if context.get("general_data_source_credential_secretkey",
None):
credentials["secretkey"] = context[
"general_data_source_credential_secretkey"]
if context.get("general_data_source_credential_endpoint", None
):
credentials["endpoint"] = context[
"general_data_source_credential_endpoint"]
credentials["bucket_in_path"] = context[
"general_data_source_credential_s3_bucket_in_path"]
credentials["ssl"] = context[
"general_data_source_credential_s3_ssl"]
credentials = credentials or None
update_data = {
"name": context["general_data_source_name"],
"description": context["general_data_source_description"],
"type": context["general_data_source_type"],
"url": context["source_url"],
"credentials": {
"user": context.get("general_data_source_credential_user",
None),
"password": context.get(
"general_data_source_credential_pass", None)
},
"credentials": credentials,
"is_public": context['general_is_public'],
"is_protected": context['general_is_protected']
}

6
sahara_dashboard/content/data_processing/jobs/templates/data_sources/_create_data_source_help.html
View File

@ -12,7 +12,11 @@
<p>
{% blocktrans %}For Data Sources on a Manila share, choose the share and enter the path relative to the share (example: /outputdir/myinputfile.txt){% endblocktrans %}
</p>
<p>
{% blocktrans %}For Data Sources in S3 storage, the URL should be specified in the format s3a://bucket/path. Also, each of the access properties may be optional if you have configured that property on your cluster manually.{% endblocktrans %}
</p>
<p>
{% blocktrans %}You may also enter an optional description for your Data Source.{% endblocktrans %}
</p>
</div>
</div>