From 4ff5370a7ec973678b642146eadd2e13e4da7f63 Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Mar 09 2018 08:52:49 +0000 Subject: distgit_directive: add param ignore_missing This is helpful when you want to download files that might or might not be there. Fixes https://pagure.io/taskotron/libtaskotron/issue/384 --- diff --git a/libtaskotron/directives/distgit_directive.py b/libtaskotron/directives/distgit_directive.py index 2c3552e..f50785e 100644 --- a/libtaskotron/directives/distgit_directive.py +++ b/libtaskotron/directives/distgit_directive.py @@ -11,6 +11,7 @@ from libtaskotron.directives import BaseDirective from libtaskotron import file_utils, python_utils from libtaskotron.ext.fedora import rpm_utils import libtaskotron.exceptions as exc +from libtaskotron.logger import log DOCUMENTATION = """ @@ -73,6 +74,12 @@ parameters: Example: ``https://src.stg.fedoraproject.org`` type: str default: https://src.fedoraproject.org + ignore_missing: + required: false + description: | + Ignore 404 error when requested files are missing in distgit + type: bool + default: False returns: | A dictionary containing following items: @@ -148,6 +155,7 @@ class DistGitDirective(BaseDirective): namespace = params.get('namespace', namespace or 'rpms') baseurl = params.get('baseurl', BASEURL) target_dir = params['target_dir'] + ignore_missing = params.get('ignore_missing', False) if not python_utils.iterable(params['path']): raise exc.TaskotronValueError("Incorrect value type of the 'path' argument: " @@ -178,8 +186,15 @@ class DistGitDirective(BaseDirective): localpath = os.path.join(target_dir, localpath) file_utils.makedirs(os.path.dirname(localpath)) url = URL_FMT.format(path=path, **format_fields) - output_data['downloaded_files'].append( - file_utils.download(url, '.', localpath) - ) + try: + output_data['downloaded_files'].append( + file_utils.download(url, '.', localpath) + ) + except exc.TaskotronRemoteError, e: + if e.errno == 404 and ignore_missing: + log.debug('File not found, ignoring: %s', url) + else: + raise e + return output_data diff --git a/libtaskotron/exceptions.py b/libtaskotron/exceptions.py index f5dd479..7dd30c8 100644 --- a/libtaskotron/exceptions.py +++ b/libtaskotron/exceptions.py @@ -35,7 +35,10 @@ class TaskotronDirectiveError(TaskotronError): class TaskotronRemoteError(TaskotronError): '''All network and remote-server related errors''' - pass + def __init__(self, e=None, errno=None): + if e: + super(TaskotronError, self).__init__(str(e)) + self.errno = errno class TaskotronRemoteTimeoutError(TaskotronRemoteError): diff --git a/libtaskotron/file_utils.py b/libtaskotron/file_utils.py index f5283eb..fdf8e50 100644 --- a/libtaskotron/file_utils.py +++ b/libtaskotron/file_utils.py @@ -128,7 +128,7 @@ def download(url, dirname, filename=None, cachedir=None): os.remove(dl_dest) except OSError, e: log.exception('Could not delete incomplete file: %s', dl_dest) - raise TaskotronRemoteError(e) + raise TaskotronRemoteError(e, errno=e.response.status_code) # create a symlink if the download was cached if cachedir: diff --git a/testing/functest_file_utils.py b/testing/functest_file_utils.py index 9be9f19..8969fe1 100644 --- a/testing/functest_file_utils.py +++ b/testing/functest_file_utils.py @@ -99,8 +99,9 @@ class TestDownload(): def test_raise(self, monkeypatch): """download errors should be raised""" + mock_response = mock.Mock(status_code=404) _download_mocked = mock.Mock(side_effect=( - requests.exceptions.RequestException('fake download failed'))) + requests.exceptions.RequestException('fake download failed', response=mock_response))) monkeypatch.setattr(file_utils, '_download', _download_mocked) with pytest.raises(exc.TaskotronRemoteError): diff --git a/testing/test_distgit_directive.py b/testing/test_distgit_directive.py index e64d94f..4cc8800 100644 --- a/testing/test_distgit_directive.py +++ b/testing/test_distgit_directive.py @@ -6,6 +6,7 @@ import os.path import pytest from dingus import Dingus +import mock from libtaskotron import file_utils from libtaskotron.directives import distgit_directive @@ -137,3 +138,45 @@ class TestDistGitDirective(): assert len(download_calls) == 1 # self.ref_nvr was not touched, so if nvr parsing gets priority, the URL will not match assert download_calls[0][1][0] == self._get_url(self.ref_path[0]) + + def test_raises_on_404(self, monkeypatch): + '''Directive must raise when file is missing in distgit (404)''' + mock_download = mock.Mock(side_effect=exc.TaskotronRemoteError('', errno=404)) + monkeypatch.setattr(file_utils, 'download', mock_download) + + with pytest.raises(exc.TaskotronRemoteError): + self.helper.process(self.ref_input, None) + + def test_ignore_404(self, monkeypatch): + '''Directive ignores missing files (404) when ignore_missing is set to True''' + mock_download = mock.Mock(side_effect=exc.TaskotronRemoteError('', errno=404)) + monkeypatch.setattr(file_utils, 'download', mock_download) + + self.ref_input['ignore_missing'] = True + + self.helper.process(self.ref_input, None) + + mock_download.assert_called_once() + + def test_raises_on_non_404_with_ignore(self, monkeypatch): + '''Directive must raise if ignore_missing is set to True but error is not 404''' + mock_download = mock.Mock(side_effect=exc.TaskotronRemoteError('', errno=503)) + monkeypatch.setattr(file_utils, 'download', mock_download) + + self.ref_input['ignore_missing'] = True + + with pytest.raises(exc.TaskotronRemoteError): + self.helper.process(self.ref_input, None) + + def test_ignore_404_on_some_files(self, monkeypatch): + '''Directive returns list of downloaded files, missing files are ignored''' + side_effects = [None, exc.TaskotronRemoteError('', errno=404)] + mock_download = mock.Mock(side_effect=side_effects) + monkeypatch.setattr(file_utils, 'download', mock_download) + + self.ref_input['ignore_missing'] = True + self.ref_input['path'] = ['foo.spec', 'foo.spek'] + + downloaded = self.helper.process(self.ref_input, None) + + assert len(downloaded) == 1