From c08f0d2662630b672a2776ee3eef8cc694f92f63 Mon Sep 17 00:00:00 2001 From: Jeremy Freudberg Date: Tue, 2 Jan 2018 03:02:16 +0000 Subject: [PATCH] Support of S3 binaries in dashboard bp sahara-support-s3 Change-Id: Ica6b5edc036fb92e8035600df51c1e074dde8093 --- .../jobs/job_binaries/forms.py | 99 +++++++++++++++++-- .../jobs/job_binaries/tests.py | 16 +++ .../job_binaries/_create_job_binary_help.html | 9 ++ 3 files changed, 117 insertions(+), 7 deletions(-) diff --git a/sahara_dashboard/content/data_processing/jobs/job_binaries/forms.py b/sahara_dashboard/content/data_processing/jobs/job_binaries/forms.py index 893104e4..a1b13c53 100644 --- a/sahara_dashboard/content/data_processing/jobs/job_binaries/forms.py +++ b/sahara_dashboard/content/data_processing/jobs/job_binaries/forms.py @@ -33,11 +33,16 @@ from saharaclient.api import base class LabeledInput(widgets.TextInput): + + def __init__(self, jb_type, *args, **kwargs): + self.jb_type = jb_type + super(LabeledInput, self).__init__(*args, **kwargs) + def render(self, name, value, attrs=None): input = super(LabeledInput, self).render(name, value, attrs) label = "%s" %\ ("id_%s_label" % name, - "swift://") + "%s://" % self.jb_type) result = "%s%s" % (label, input) return mark_safe(result) @@ -67,6 +72,7 @@ class JobBinaryCreateForm(forms.SelfHandlingForm): label=_("URL"), required=False, widget=LabeledInput( + "swift", attrs={ 'class': 'switched', 'data-switch-on': 'jb_type', @@ -156,6 +162,48 @@ class JobBinaryCreateForm(forms.SelfHandlingForm): 'data-jb_type-swift': _('Password') })) + self.fields["job_binary_s3_url"] = forms.CharField( + label=_("URL"), + required=False, + widget=LabeledInput( + "s3", + attrs={ + 'class': 'switched', + 'data-switch-on': 'jb_type', + 'data-jb_type-s3': _('URL') + })) + + self.fields["job_binary_s3_endpoint"] = forms.CharField( + label=_("S3 Endpoint"), + required=False, + widget=forms.TextInput( + attrs={ + 'class': 'switched', + 'data-switch-on': 'jb_type', + 'data-jb_type-s3': _('S3 Endpoint') + })) + + self.fields["job_binary_access_key"] = forms.CharField( + label=_("Access Key"), + required=False, + widget=forms.TextInput( + attrs={ + 'class': 'switched', + 'data-switch-on': 'jb_type', + 'data-jb_type-s3': _('Access Key') + })) + + self.fields["job_binary_secret_key"] = forms.CharField( + label=_("Secret Key"), + required=False, + widget=forms.PasswordInput( + attrs={ + 'autocomplete': 'off', + 'class': 'switched', + 'data-switch-on': 'jb_type', + 'data-jb_type-s3': _('Secret Key') + })) + self.fields["job_binary_description"] = ( forms.CharField(label=_("Description"), required=False, @@ -168,7 +216,8 @@ class JobBinaryCreateForm(forms.SelfHandlingForm): self.fields["job_binary_type"].choices =\ [("internal-db", "Internal database"), - ("swift", "Swift")] + ("swift", "Swift"), + ("s3", "S3")] self.fields["job_binary_internal"].choices =\ self.populate_job_binary_internal_choices(request) @@ -185,7 +234,7 @@ class JobBinaryCreateForm(forms.SelfHandlingForm): jb = self.initial["job_binary"] for field in self.fields: if self.FIELD_MAP[field]: - if field == "job_binary_url": + if field in ["job_binary_url", "job_binary_s3_url"]: url = getattr(jb, self.FIELD_MAP[field], None) self.set_initial_values_by_type(url) else: @@ -201,6 +250,9 @@ class JobBinaryCreateForm(forms.SelfHandlingForm): elif parsed.scheme == "swift": self.fields["job_binary_url"].initial = ( "{0}{1}".format(parsed.netloc, parsed.path)) + elif parsed.scheme == "s3": + self.fields["job_binary_s3_url"].initial = ( + "{0}{1}".format(parsed.netloc, parsed.path)) else: self.fields["job_binary_url"].initial = "{0}".format(parsed.netloc) @@ -233,12 +285,18 @@ class JobBinaryCreateForm(forms.SelfHandlingForm): extra = {} jb_type = context.get("job_binary_type") - bin_url = "%s://%s" % (context["job_binary_type"], - context["job_binary_url"]) + if jb_type != "s3": + bin_url = "%s://%s" % (context["job_binary_type"], + context["job_binary_url"]) + else: + bin_url = "%s://%s" % (context["job_binary_type"], + context["job_binary_s3_url"]) if(jb_type == "internal-db"): bin_url = self.handle_internal(request, context) elif(jb_type == "swift"): extra = self.handle_swift(request, context) + elif(jb_type == "s3"): + extra = self.handle_s3(request, context) elif(jb_type == "manila"): bin_url = "%s://%s%s" % (context["job_binary_type"], context["job_binary_manila_share"], @@ -326,6 +384,21 @@ class JobBinaryCreateForm(forms.SelfHandlingForm): } return extra + def handle_s3(self, request, context): + accesskey = context['job_binary_access_key'] + secretkey = context['job_binary_secret_key'] + endpoint = context['job_binary_s3_endpoint'] + + extra = {} + if accesskey != "": + extra["accesskey"] = accesskey + if secretkey != "": + extra["secretkey"] = secretkey + if endpoint != "": + extra["endpoint"] = endpoint + + return extra + def get_unique_binary_name(self, request, base_name): try: internals = saharaclient.job_binary_internal_list(request) @@ -350,9 +423,13 @@ class JobBinaryEditForm(JobBinaryCreateForm): 'job_binary_script_name': None, 'job_binary_type': None, 'job_binary_url': 'url', + 'job_binary_s3_url': 'url', 'job_binary_username': None, 'job_binary_manila_share': None, 'job_binary_manila_path': None, + 'job_binary_access_key': None, + 'job_binary_secret_key': None, + 'job_binary_s3_endpoint': None, 'is_public': 'is_public', 'is_protected': 'is_protected', } @@ -360,11 +437,19 @@ class JobBinaryEditForm(JobBinaryCreateForm): def handle(self, request, context): try: extra = {} - bin_url = "%s://%s" % (context["job_binary_type"], - context["job_binary_url"]) + if context["job_binary_type"] != "s3": + bin_url = "%s://%s" % (context["job_binary_type"], + context["job_binary_url"]) + else: + bin_url = "%s://%s" % (context["job_binary_type"], + context["job_binary_s3_url"]) + if (context["job_binary_type"] == "swift"): extra = self.handle_swift(request, context) + if (context["job_binary_type"] == "s3"): + extra = self.handle_s3(request, context) + update_data = { "name": context["job_binary_name"], "description": context["job_binary_description"], diff --git a/sahara_dashboard/content/data_processing/jobs/job_binaries/tests.py b/sahara_dashboard/content/data_processing/jobs/job_binaries/tests.py index ed403330..372e30fc 100644 --- a/sahara_dashboard/content/data_processing/jobs/job_binaries/tests.py +++ b/sahara_dashboard/content/data_processing/jobs/job_binaries/tests.py @@ -152,3 +152,19 @@ class DataProcessingJobBinaryTests(test.TestCase): res = self.client.post(CREATE_URL, form_data) self.assertNoFormErrors(res) + + @test.create_stubs({api.sahara: ('job_binary_create', + 'job_binary_internal_list')}) + def test_create_s3(self): + form_data = { + "job_binary_type": "s3", + "job_binary_s3_url": "s3://a/b", + "job_binary_access_key": "acc", + "job_binary_secret_key": "sec", + "job_binary_s3_endpoint": "http://pointy.end", + "job_binary_name": "tests3", + "job_binary_description": "Test s3 description" + } + self.mox.ReplayAll() + res = self.client.post(CREATE_URL, form_data) + self.assertNoFormErrors(res) diff --git a/sahara_dashboard/content/data_processing/jobs/templates/job_binaries/_create_job_binary_help.html b/sahara_dashboard/content/data_processing/jobs/templates/job_binaries/_create_job_binary_help.html index e08aa2f2..c3653a9e 100644 --- a/sahara_dashboard/content/data_processing/jobs/templates/job_binaries/_create_job_binary_help.html +++ b/sahara_dashboard/content/data_processing/jobs/templates/job_binaries/_create_job_binary_help.html @@ -9,6 +9,7 @@

@@ -26,6 +27,14 @@

  • {% blocktrans %}Enter the username and password required to access that file{% endblocktrans %}

    • +

    + {% blocktrans %}For S3 job binaries, you must:{% endblocktrans %} +

    +

    {% blocktrans %}You may also enter an optional description for your job binary.{% endblocktrans %}