From fdeeaf0ad889f63f3d99af365cfe9ad041fa9bda Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Apr 13 2018 12:54:29 +0000 Subject: [PATCH 1/2] Fix unicode issue for update command in Python 3 With this patch, in update command, unicode string is used consistently through the whole process. String written into file by write() method is handled carefully with or without encoding to byte string for Python 2 and 3 individually. Issue reported in #206 was not caught due to write() was mocked. This patch also fixes this problem. Now, during the test, real bodhi.template and clog file are written into and read from file system. Tests are updated accordingly. Fixes #206 Signed-off-by: Chenxiong Qi --- diff --git a/fedpkg/cli.py b/fedpkg/cli.py index 34e7857..23945ff 100644 --- a/fedpkg/cli.py +++ b/fedpkg/cli.py @@ -14,6 +14,7 @@ from __future__ import print_function from pyrpkg.cli import cliClient import argparse import hashlib +import io import os import re import json @@ -286,14 +287,17 @@ close_bugs=True suggest_reboot=False """ - bodhi_args = {'nvr': self.cmd.nvr, - 'bugs': '', - 'descr': 'Here is where you give an explanation' - ' of your update.'} + bodhi_args = { + 'nvr': self.cmd.nvr, + 'bugs': six.u(''), + 'descr': six.u( + 'Here is where you give an explanation of your update.') + } # Extract bug numbers from the latest changelog entry self.cmd.clog() - with open('clog', 'r') as f: + clog_file = os.path.join(self.cmd.path, 'clog') + with io.open(clog_file, encoding='utf-8') as f: clog = f.read() bugs = re.findall(r'#([0-9]*)', clog) if bugs: @@ -303,12 +307,6 @@ suggest_reboot=False bodhi_args['descr'], bodhi_args['changelog'] = \ self._format_update_clog(clog) - if six.PY2: - # log may contain unicode characters, convert log to unicode string - # to ensure text can be wrapped correctly in follow step. - bodhi_args['descr'] = bodhi_args['descr'].decode('utf-8') - bodhi_args['changelog'] = bodhi_args['changelog'].decode('utf-8') - template = textwrap.dedent(template) % bodhi_args # Calculate the hash of the unaltered template @@ -317,8 +315,8 @@ suggest_reboot=False orig_hash = orig_hash.hexdigest() # Write out the template - with open('bodhi.template', 'w') as f: - f.write(template.encode('utf-8')) + with io.open('bodhi.template', 'w', encoding='utf-8') as f: + f.write(template) # Open the template in a text editor editor = os.getenv('EDITOR', 'vi') diff --git a/test/test_cli.py b/test/test_cli.py index 7662d44..b991dd4 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -10,6 +10,8 @@ # option) any later version. See http://www.gnu.org/copyleft/gpl.html for # the full text of the license. +import io +import os import sys import json from datetime import datetime, timedelta @@ -25,7 +27,7 @@ from pyrpkg.errors import rpkgError from utils import CliTestCase from fedpkg.bugzilla import BugzillaClient -from mock import call, patch, mock_open, PropertyMock, Mock +from mock import call, patch, PropertyMock, Mock class TestUpdate(CliTestCase): @@ -36,7 +38,7 @@ class TestUpdate(CliTestCase): self.nvr_patcher = patch('fedpkg.Commands.nvr', new_callable=PropertyMock, - return_value='fedpkg-1.29-9') + return_value='fedpkg-1.29-9.fc27') self.mock_nvr = self.nvr_patcher.start() self.run_command_patcher = patch('fedpkg.Commands._run_command') @@ -55,13 +57,20 @@ class TestUpdate(CliTestCase): self.os_environ_patcher = patch.dict('os.environ', {'EDITOR': 'vi'}) self.os_environ_patcher.start() - self.fake_clog = '\n'.join([ + self.fake_clog = list(six.moves.map(six.u, [ 'Add tests for command update', 'New command update - #1000', - 'Fix tests - #2000' - ]) + 'Fix tests - #2000', + '处理一些Unicode字符číář', + ])) + clog_file = os.path.join(self.cloned_repo_path, 'clog') + with io.open(clog_file, 'w', encoding='utf-8') as f: + f.write(os.linesep.join(self.fake_clog)) def tearDown(self): + if os.path.exists('bodhi.template'): + os.unlink('bodhi.template') + os.unlink(os.path.join(self.cloned_repo_path, 'clog')) self.os_environ_patcher.stop() self.clog_patcher.stop() self.get_bodhi_version_patcher.stop() @@ -73,17 +82,24 @@ class TestUpdate(CliTestCase): with patch('sys.argv', new=cli_cmd): return self.new_cli(name=name, cfg=cfg) - def create_bodhi_update(self, cli): - mocked_open = mock_open(read_data=self.fake_clog) - with patch('six.moves.builtins.open', mocked_open): - with patch('os.unlink') as unlink: - cli.update() - - # Ensure these files are removed in the end - unlink.assert_has_calls([ - call('bodhi.template'), - call('clog') - ]) + def assert_bodhi_update(self, cli): + with patch('os.unlink') as unlink: + cli.update() + + unlink.assert_has_calls([ + call('bodhi.template'), + call('clog') + ]) + + with io.open('bodhi.template', encoding='utf-8') as f: + bodhi_template = f.read() + self.assertTrue(self.mock_nvr.return_value in bodhi_template) + self.assertTrue('1000,2000' in bodhi_template) + self.assertTrue(self.fake_clog[0] in bodhi_template) + rest_clog = os.linesep.join([ + six.u('# {0}').format(line) for line in self.fake_clog[1:] + ]) + self.assertTrue(rest_clog in bodhi_template) def test_fail_if_missing_config_options(self): cli_cmd = ['fedpkg', '--path', self.cloned_repo_path, 'update'] @@ -106,7 +122,7 @@ class TestUpdate(CliTestCase): cli = self.get_cli(cli_cmd) six.assertRaisesRegex( self, rpkgError, 'No bodhi update details saved', - self.create_bodhi_update, cli) + self.assert_bodhi_update, cli) self.mock_run_command.assert_called_once_with( ['vi', 'bodhi.template'], shell=True) @@ -123,7 +139,7 @@ class TestUpdate(CliTestCase): cli_cmd = ['fedpkg', '--path', self.cloned_repo_path, 'update'] cli = self.get_cli(cli_cmd) - self.create_bodhi_update(cli) + self.assert_bodhi_update(cli) self.mock_run_command.assert_has_calls([ call(['vi', 'bodhi.template'], shell=True), @@ -146,7 +162,7 @@ class TestUpdate(CliTestCase): cli = self.get_cli(cli_cmd) six.assertRaisesRegex( self, rpkgError, 'Could not generate update request', - self.create_bodhi_update, cli) + self.assert_bodhi_update, cli) @patch('os.path.isfile', return_value=True) @patch('hashlib.new') @@ -163,7 +179,7 @@ class TestUpdate(CliTestCase): cli = self.get_cli(cli_cmd) six.assertRaisesRegex( self, rpkgError, 'This system has bodhi v4, which is unsupported', - self.create_bodhi_update, cli) + self.assert_bodhi_update, cli) @patch('os.path.isfile', return_value=True) @patch('hashlib.new') @@ -180,7 +196,7 @@ class TestUpdate(CliTestCase): cli = self.get_cli(cli_cmd, name='fedpkg-stage', cfg='fedpkg-stage.conf') - self.create_bodhi_update(cli) + self.assert_bodhi_update(cli) self.mock_run_command.assert_has_calls([ call(['vi', 'bodhi.template'], shell=True), From 045350dfd328cd3e273efda75fa8d83e05537bce Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Apr 13 2018 12:54:29 +0000 Subject: [PATCH 2/2] Copy pip-pycurl to ensure pycurl is installed correctly rpkg can be installed from PyPI while running tests by tox. pip-pycurl is copied from rpkg to ensure pycurl is installed correctly in different Fedora distro 26 and >=27. Refer to https://fedoraproject.org/wiki/Changes/libcurlBackToOpenSSL Signed-off-by: Chenxiong Qi --- diff --git a/MANIFEST.in b/MANIFEST.in index cfe82df..e721ea8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include doc/fedpkg_man_page.py include doc/release-guide.markdown include test/utils.py include test *.conf +include pip-pycurl recursive-include conf * include tox.ini include requirements.txt tests-requirements.txt diff --git a/pip-pycurl b/pip-pycurl new file mode 100755 index 0000000..b63ea3c --- /dev/null +++ b/pip-pycurl @@ -0,0 +1,19 @@ +#!/bin/bash + +if python -c "import pycurl" &>/dev/null; then + exit 0 +fi + +# We need to build pycurl with openssl in Fedora 27, otherwise nss should be +# used. +# See also: https://fedoraproject.org/wiki/Changes/libcurlBackToOpenSSL + +dist=$(rpm --eval "%{dist}") +dist=${dist:3} + +if [ $dist -ge 27 ]; then + install_option="--with-openssl" +else + install_option="--with-nss" +fi +pip install -v -I --install-option="${install_option}" "pycurl>=7.19" diff --git a/tox.ini b/tox.ini index 942339f..9541c0e 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/tests-requirements.txt commands = - pip install -I --install-option="--with-openssl" "pycurl>=7.19" + {toxinidir}/pip-pycurl nosetests {posargs} [testenv:py26]