Support of S3 data sources in dashboard
Change-Id: Iad4c0739f774b4f87ec6f24211c3f39eec7a7582
This commit is contained in:
parent
56b69d6477
commit
43c1c36c0d
|
@ -417,16 +417,18 @@ def cluster_update_acl_rules(request, cl_id, is_public=None,
|
|||
|
||||
def data_source_create(request, name, description, ds_type, url,
|
||||
credential_user=None, credential_pass=None,
|
||||
is_public=None, is_protected=None):
|
||||
is_public=None, is_protected=None,
|
||||
s3_credentials=None):
|
||||
return client(request).data_sources.create(
|
||||
name=name,
|
||||
description=description,
|
||||
data_source_type=ds_type,
|
||||
url=url,
|
||||
credential_user=credential_user,
|
||||
credential_pass=credential_pass,
|
||||
credential_user=credential_user or None,
|
||||
credential_pass=credential_pass or None,
|
||||
is_public=is_public,
|
||||
is_protected=is_protected)
|
||||
is_protected=is_protected,
|
||||
s3_credentials=s3_credentials)
|
||||
|
||||
|
||||
def data_source_list(request, search_opts=None, limit=None, marker=None):
|
||||
|
|
|
@ -92,10 +92,11 @@ class DataProcessingDataSourceTests(test.TestCase):
|
|||
data_source.description,
|
||||
data_source.type,
|
||||
data_source.url,
|
||||
"",
|
||||
"",
|
||||
'',
|
||||
'',
|
||||
is_public=False,
|
||||
is_protected=False)
|
||||
is_protected=False,
|
||||
s3_credentials=None)
|
||||
|
||||
self.assertNoFormErrors(res)
|
||||
self.assertRedirectsNoFollow(res, INDEX_URL)
|
||||
|
@ -164,6 +165,32 @@ class DataProcessingDataSourceTests(test.TestCase):
|
|||
IsA(six.text_type),
|
||||
IsA(six.text_type),
|
||||
IsA(str),
|
||||
"", "", is_public=False, is_protected=False)
|
||||
'', '', is_public=False, is_protected=False, s3_credentials=None)
|
||||
self.mock_share_list.assert_called_once_with(IsHttpRequest())
|
||||
self.assertNoFormErrors(res)
|
||||
|
||||
@test.create_mocks({api.sahara: ('data_source_create',)})
|
||||
def test_create_s3(self):
|
||||
form_data = {
|
||||
"data_source_type": "s3",
|
||||
"data_source_url": "s3a://a/b",
|
||||
"data_source_credential_accesskey": "acc",
|
||||
"data_source_credential_secretkey": "sec",
|
||||
"data_source_credential_endpoint": "pointy.end",
|
||||
"data_source_credential_s3_bucket_in_url": False,
|
||||
"data_source_credential_s3_ssl": True,
|
||||
"data_source_name": "tests3",
|
||||
"data_source_description": "Test s3 description"
|
||||
}
|
||||
res = self.client.post(CREATE_URL, form_data)
|
||||
self.mock_data_source_create.return_value = True
|
||||
self.mock_data_source_create.assert_called_once_with(
|
||||
IsHttpRequest(),
|
||||
IsA(six.text_type),
|
||||
IsA(six.text_type),
|
||||
IsA(six.text_type),
|
||||
IsA(six.text_type),
|
||||
IsA(six.text_type),
|
||||
IsA(six.text_type),
|
||||
is_public=False, is_protected=False, s3_credentials=IsA(dict))
|
||||
self.assertNoFormErrors(res)
|
||||
|
|
|
@ -52,7 +52,8 @@ class GeneralConfigAction(workflows.Action):
|
|||
"data-ds_type-manila": _("Path on share"),
|
||||
"data-ds_type-swift": _("URL"),
|
||||
"data-ds_type-hdfs": _("URL"),
|
||||
"data-ds_type-maprfs": _("URL")
|
||||
"data-ds_type-maprfs": _("URL"),
|
||||
"data-ds_type-s3": _("URL"),
|
||||
}))
|
||||
|
||||
data_source_credential_user = forms.CharField(
|
||||
|
@ -74,6 +75,56 @@ class GeneralConfigAction(workflows.Action):
|
|||
label=_("Source password"),
|
||||
required=False)
|
||||
|
||||
data_source_credential_accesskey = forms.CharField(
|
||||
label=_("S3 accsss key"),
|
||||
required=False,
|
||||
widget=forms.TextInput(attrs={
|
||||
"class": "switched",
|
||||
"data-switch-on": "ds_type",
|
||||
"data-ds_type-s3": _("S3 access key")
|
||||
}))
|
||||
|
||||
data_source_credential_secretkey = forms.CharField(
|
||||
widget=forms.PasswordInput(attrs={
|
||||
'class': 'switched',
|
||||
'data-switch-on': 'ds_type',
|
||||
'data-ds_type-s3': _("S3 secret key"),
|
||||
'autocomplete': 'off'
|
||||
}),
|
||||
label=_("S3 secret key"),
|
||||
required=False)
|
||||
|
||||
data_source_credential_endpoint = forms.CharField(
|
||||
label=_("S3 endpoint"),
|
||||
required=False,
|
||||
help_text=_("Endpoint should be specified without protocol"),
|
||||
widget=forms.TextInput(attrs={
|
||||
"class": "switched",
|
||||
"data-switch-on": "ds_type",
|
||||
"data-ds_type-s3": _("S3 endpoint")
|
||||
}))
|
||||
|
||||
data_source_credential_s3_ssl = forms.BooleanField(
|
||||
label=_("Use SSL"),
|
||||
required=False,
|
||||
initial=False,
|
||||
widget=forms.CheckboxInput(attrs={
|
||||
"class": "switched",
|
||||
"data-switch-on": "ds_type",
|
||||
"data-ds_type-s3": _("Use SSL")
|
||||
}))
|
||||
|
||||
data_source_credential_s3_bucket_in_path = forms.BooleanField(
|
||||
label=_("Use bucket in path"),
|
||||
required=False,
|
||||
initial=True,
|
||||
help_text=_("If checked, bucket will be in path instead of host"),
|
||||
widget=forms.CheckboxInput(attrs={
|
||||
"class": "switched",
|
||||
"data-switch-on": "ds_type",
|
||||
"data-ds_type-s3": _("Use bucket in path")
|
||||
}))
|
||||
|
||||
data_source_description = forms.CharField(
|
||||
label=_("Description"),
|
||||
required=False,
|
||||
|
@ -87,7 +138,8 @@ class GeneralConfigAction(workflows.Action):
|
|||
|
||||
self.fields["data_source_type"].choices = [("swift", "Swift"),
|
||||
("hdfs", "HDFS"),
|
||||
("maprfs", "MapR FS")]
|
||||
("maprfs", "MapR FS"),
|
||||
("s3", "S3")]
|
||||
# If Manila is running, enable it as a choice for a data source
|
||||
if saharaclient.base.is_service_enabled(request, 'share'):
|
||||
self.fields["data_source_type"].choices.append(
|
||||
|
@ -142,6 +194,22 @@ class CreateDataSource(workflows.Workflow):
|
|||
default_steps = (GeneralConfig, )
|
||||
|
||||
def handle(self, request, context):
|
||||
s3_credentials = {}
|
||||
if context["general_data_source_type"] == "s3":
|
||||
if context.get("general_data_source_credential_accesskey", None):
|
||||
s3_credentials["accesskey"] = context[
|
||||
"general_data_source_credential_accesskey"]
|
||||
if context.get("general_data_source_credential_secretkey", None):
|
||||
s3_credentials["secretkey"] = context[
|
||||
"general_data_source_credential_secretkey"]
|
||||
if context.get("general_data_source_credential_endpoint", None):
|
||||
s3_credentials["endpoint"] = context[
|
||||
"general_data_source_credential_endpoint"]
|
||||
s3_credentials["bucket_in_path"] = context[
|
||||
"general_data_source_credential_s3_bucket_in_path"]
|
||||
s3_credentials["ssl"] = context[
|
||||
"general_data_source_credential_s3_ssl"]
|
||||
s3_credentials = s3_credentials or None
|
||||
try:
|
||||
self.object = saharaclient.data_source_create(
|
||||
request,
|
||||
|
@ -152,7 +220,8 @@ class CreateDataSource(workflows.Workflow):
|
|||
context.get("general_data_source_credential_user", None),
|
||||
context.get("general_data_source_credential_pass", None),
|
||||
is_public=context['general_is_public'],
|
||||
is_protected=context['general_is_protected']
|
||||
is_protected=context['general_is_protected'],
|
||||
s3_credentials=s3_credentials
|
||||
)
|
||||
|
||||
hlps = helpers.Helpers(request)
|
||||
|
|
|
@ -37,6 +37,11 @@ class EditDataSource(create.CreateDataSource):
|
|||
"data_source_url": "url",
|
||||
"data_source_credential_user": None,
|
||||
"data_source_credential_pass": None,
|
||||
"data_source_credential_accesskey": None,
|
||||
"data_source_credential_secretkey": None,
|
||||
"data_source_credential_endpoint": None,
|
||||
"data_source_credential_s3_ssl": None,
|
||||
"data_source_credential_s3_bucket_in_path": None,
|
||||
"data_source_manila_share": None,
|
||||
'is_public': "is_public",
|
||||
'is_protected': "is_protected"
|
||||
|
@ -72,17 +77,36 @@ class EditDataSource(create.CreateDataSource):
|
|||
|
||||
def handle(self, request, context):
|
||||
try:
|
||||
credentials = {}
|
||||
if context["general_data_source_type"] == "swift":
|
||||
credentials["user"] = context.get(
|
||||
"general_data_source_credential_user", None)
|
||||
credentials["password"] = context.get(
|
||||
"general_data_source_credential_pass", None)
|
||||
elif context["general_data_source_type"] == "s3":
|
||||
if context.get("general_data_source_credential_accesskey",
|
||||
None):
|
||||
credentials["accesskey"] = context[
|
||||
"general_data_source_credential_accesskey"]
|
||||
if context.get("general_data_source_credential_secretkey",
|
||||
None):
|
||||
credentials["secretkey"] = context[
|
||||
"general_data_source_credential_secretkey"]
|
||||
if context.get("general_data_source_credential_endpoint", None
|
||||
):
|
||||
credentials["endpoint"] = context[
|
||||
"general_data_source_credential_endpoint"]
|
||||
credentials["bucket_in_path"] = context[
|
||||
"general_data_source_credential_s3_bucket_in_path"]
|
||||
credentials["ssl"] = context[
|
||||
"general_data_source_credential_s3_ssl"]
|
||||
credentials = credentials or None
|
||||
update_data = {
|
||||
"name": context["general_data_source_name"],
|
||||
"description": context["general_data_source_description"],
|
||||
"type": context["general_data_source_type"],
|
||||
"url": context["source_url"],
|
||||
"credentials": {
|
||||
"user": context.get("general_data_source_credential_user",
|
||||
None),
|
||||
"password": context.get(
|
||||
"general_data_source_credential_pass", None)
|
||||
},
|
||||
"credentials": credentials,
|
||||
"is_public": context['general_is_public'],
|
||||
"is_protected": context['general_is_protected']
|
||||
}
|
||||
|
|
|
@ -12,7 +12,11 @@
|
|||
<p>
|
||||
{% blocktrans %}For Data Sources on a Manila share, choose the share and enter the path relative to the share (example: /outputdir/myinputfile.txt){% endblocktrans %}
|
||||
</p>
|
||||
<p>
|
||||
{% blocktrans %}For Data Sources in S3 storage, the URL should be specified in the format s3a://bucket/path. Also, each of the access properties may be optional if you have configured that property on your cluster manually.{% endblocktrans %}
|
||||
</p>
|
||||
|
||||
<p>
|
||||
{% blocktrans %}You may also enter an optional description for your Data Source.{% endblocktrans %}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
Loading…
Reference in New Issue