From ca356215ce841ff8b64890474b20ba75ff2cef0f Mon Sep 17 00:00:00 2001 From: Guillaume DeMengin Date: Sat, 26 Nov 2022 19:35:37 +0100 Subject: [PATCH] get_build_artifact_as_bytes to support non-json files new function get_build_artifact_as_bytes to replace get_build_artifact (incompatible with artifacts not json-formatted) + use a stream to download binary artifacts all artifacts are returned as bytes to avoid encoding issues and an exception NotFoundException is raised in case of missing artifact Closes-Bug: #1973243 Change-Id: I24ce4ecd854f8a19ed4d760404adb7d1ac6b5509 --- jenkins/__init__.py | 37 +++++++++++++-- tests/test_build.py | 110 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 142 insertions(+), 5 deletions(-) diff --git a/jenkins/__init__.py b/jenkins/__init__.py index 679fe53..cdedb79 100755 --- a/jenkins/__init__.py +++ b/jenkins/__init__.py @@ -544,13 +544,13 @@ class Jenkins(object): # when accessing .text property return response - def _request(self, req): + def _request(self, req, stream=None): r = self._session.prepare_request(req) # requests.Session.send() does not honor env settings by design # see https://github.com/requests/requests/issues/2807 _settings = self._session.merge_environment_settings( - r.url, {}, None, self._session.verify, None) + r.url, {}, stream, self._session.verify, None) _settings['timeout'] = self.timeout return self._session.send(r, **_settings) @@ -561,7 +561,14 @@ class Jenkins(object): ''' return self.jenkins_request(req, add_crumb, resolve_auth).text - def jenkins_request(self, req, add_crumb=True, resolve_auth=True): + def jenkins_open_stream(self, req, add_crumb=True, resolve_auth=True): + '''Return the HTTP response body from a ``requests.Request``. + + :returns: ``stream`` + ''' + return self.jenkins_request(req, add_crumb, resolve_auth, True) + + def jenkins_request(self, req, add_crumb=True, resolve_auth=True, stream=None): '''Utility routine for opening an HTTP request to a Jenkins server. :param req: A ``requests.Request`` to submit. @@ -569,6 +576,7 @@ class Jenkins(object): before submitting. Defaults to ``True``. :param resolve_auth: If True, maybe add authentication. Defaults to ``True``. + :param stream: If True, return a stream :returns: A ``requests.Response`` object. ''' try: @@ -578,7 +586,7 @@ class Jenkins(object): self.maybe_add_crumb(req) return self._response_handler( - self._request(req)) + self._request(req, stream)) except req_exc.HTTPError as e: # Jenkins's funky authentication means its nigh impossible to @@ -742,6 +750,27 @@ class Jenkins(object): # This can happen if the artifact is not found return None + def get_build_artifact_as_bytes(self, name, number, artifact): + """Get artifacts from job + + :param name: Job name, ``str`` + :param number: Build number, ``str`` (also accepts ``int``) + :param artifact: Artifact relative path, ``str`` + :returns: artifact to download, ``bytes`` + """ + folder_url, short_name = self._get_job_folder(name) + + try: + with self.jenkins_open_stream(requests.Request( + 'GET', self._build_url(BUILD_ARTIFACT, locals()))) as response: + if response.encoding is None: + return response.raw.read() + else: + return response.text.encode(response.encoding) + raise JenkinsException('job[%s] number[%s] does not exist' % (name, number)) + except requests.exceptions.HTTPError: + raise JenkinsException('job[%s] number[%s] does not exist' % (name, number)) + def get_build_stages(self, name, number): """Get stages info from job diff --git a/tests/test_build.py b/tests/test_build.py index acd3a58..616eaa6 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1,7 +1,7 @@ import json import collections -from mock import patch +from mock import patch, Mock import jenkins from tests.base import JenkinsTestBase @@ -812,6 +812,114 @@ class JenkinsBuildArtifactUrlTest(JenkinsTestBase): 'Error in request. Possibly authentication failed [401]: Not Authorised') +class JenkinsBuildArtifactAsBytesUrlTest(JenkinsTestBase): + + def streamMock(self, encoding=None, text=None, binary=None): + streamMock = Mock() + streamMock.__exit__ = Mock() + streamMock.__enter__ = Mock() + if encoding is None and text is None and binary is None: + streamMock.__enter__.return_value = None + return streamMock + streamMock.__enter__.return_value = Mock() + streamMock.__enter__.return_value.encoding = encoding + streamMock.__enter__.return_value.text = text + streamMock.__enter__.return_value.raw = Mock() + streamMock.__enter__.return_value.raw.read = Mock() + streamMock.__enter__.return_value.raw.read.return_value = binary + return streamMock + + @patch.object(jenkins.Jenkins, 'jenkins_open_stream') + def test_simple_ascii_artifact(self, jenkins_mock): + jenkins_mock.return_value = self.streamMock('utf-8', 'ascii') + ret = self.j.get_build_artifact_as_bytes(u'Test Job', number='52', artifact="filename") + self.assertEqual(ret, b'ascii') + self.assertEqual( + jenkins_mock.call_args[0][0].url, + self.make_url('job/Test%20Job/52/artifact/filename')) + self._check_requests(jenkins_mock.call_args_list) + + @patch.object(jenkins.Jenkins, 'jenkins_open_stream') + def test_simple_binary_artifact(self, jenkins_mock): + jenkins_mock.return_value = self.streamMock(binary=b'\0\1\2') + ret = self.j.get_build_artifact_as_bytes(u'Test Job', number='52', artifact="filename") + self.assertEqual(ret, b'\0\1\2') + self.assertEqual( + jenkins_mock.call_args[0][0].url, + self.make_url('job/Test%20Job/52/artifact/filename')) + self._check_requests(jenkins_mock.call_args_list) + + @patch.object(jenkins.Jenkins, 'jenkins_open_stream') + def test_in_folder(self, jenkins_mock): + jenkins_mock.return_value = self.streamMock('utf-8', 'ascii') + ret = self.j.get_build_artifact_as_bytes(u'a Folder/Test Job', number='52', artifact="file name") + self.assertEqual(ret, b'ascii') + self.assertEqual( + jenkins_mock.call_args[0][0].url, + self.make_url('job/a%20Folder/job/Test%20Job/52/artifact/file%20name')) + self._check_requests(jenkins_mock.call_args_list) + + @patch.object(jenkins.Jenkins, 'jenkins_open_stream') + def test_matrix(self, jenkins_mock): + jenkins_mock.return_value = self.streamMock('utf-8', 'ascii') + ret = self.j.get_build_artifact_as_bytes(u'a Folder/Test Job', number='52/index=matrix', + artifact="file name") + self.assertEqual(ret, b'ascii') + self.assertEqual( + jenkins_mock.call_args[0][0].url, + self.make_url('job/a%20Folder/job/Test%20Job/52/index=matrix/artifact/file%20name')) + self._check_requests(jenkins_mock.call_args_list) + + @patch('jenkins.requests.Session.send', autospec=True) + def test_404_item_not_found(self, session_send_mock): + session_send_mock.side_effect = iter([ + build_response_mock(404, reason="Not Found"), # crumb + build_response_mock(404, reason="Not Found"), # request + ]) + with self.assertRaises(jenkins.JenkinsException) as context_manager: + self.j.get_build_artifact_as_bytes(u'TestJob', number='52', artifact="filename") + self.assertEqual( + str(context_manager.exception), + 'Requested item could not be found') + + @patch.object(jenkins.Jenkins, 'jenkins_open_stream') + def test_open_return_none(self, jenkins_mock): + jenkins_mock.return_value = self.streamMock() + + with self.assertRaises(jenkins.JenkinsException) as context_manager: + self.j.get_build_artifact_as_bytes(u'TestJob', number='52', artifact="filename") + self.assertEqual( + str(context_manager.exception), + 'job[TestJob] number[52] does not exist') + self._check_requests(jenkins_mock.call_args_list) + + @patch('jenkins.requests.Session.send', autospec=True) + def test_raise_HTTPError(self, session_send_mock): + session_send_mock.side_effect = iter([ + build_response_mock(401, reason="Not Authorised"), # crumb + build_response_mock(401, reason="Not Authorised"), # request + ]) + + with self.assertRaises(jenkins.JenkinsException) as context_manager: + self.j.get_build_artifact_as_bytes(u'TestJob', number='52', artifact="filename") + self.assertEqual( + str(context_manager.exception), + 'Error in request. Possibly authentication failed [401]: Not Authorised') + + @patch('jenkins.requests.Session.send', autospec=True) + def test_in_folder_raise_HTTPError(self, session_send_mock): + session_send_mock.side_effect = iter([ + build_response_mock(401, reason="Not Authorised"), # crumb + build_response_mock(401, reason="Not Authorised"), # request + ]) + + with self.assertRaises(jenkins.JenkinsException) as context_manager: + self.j.get_build_artifact_as_bytes(u'a Folder/TestJob', number='52', artifact="filename") + self.assertEqual( + str(context_manager.exception), + 'Error in request. Possibly authentication failed [401]: Not Authorised') + + class JenkinsBuildStagesUrlTest(JenkinsTestBase): @patch.object(jenkins.Jenkins, 'jenkins_open')