From 4320ec476f62af6c6fc3709db58f0038be447e94 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Nov 20 2017 16:47:55 +0000 Subject: [PATCH 1/2] Add coverage capturing to test suite Signed-off-by: Patrick Uiterwijk --- diff --git a/ipsilon/install/ipsilon-server-install b/ipsilon/install/ipsilon-server-install index 1a4efcb..0e1c8f9 100755 --- a/ipsilon/install/ipsilon-server-install +++ b/ipsilon/install/ipsilon-server-install @@ -95,6 +95,11 @@ def install(plugins, args): shutil.move(idp_conf, '%s.backup.%s' % (idp_conf, now)) if not os.path.exists(instance_conf): os.makedirs(instance_conf, 0700) + coverage = '' + if args['coverage'] or args['branch_coverage']: + coverage += 'coverage.enabled = True' + if args['coverage_branch']: + coverage += '\ncoverage.branch = True' confopts = {'instance': args['instance'], 'instanceurl': args['instanceurl'], 'needs_mount': args.get('needs_mount'), @@ -118,7 +123,8 @@ def install(plugins, args): 'database_url'] % {'datadir': args['data_dir'], 'dbname': 'saml2sessions'}, 'secure': "False" if args['secure'] == "no" else "True", - 'debugging': "True" if args['server_debugging'] else "False"} + 'debugging': "True" if args['server_debugging'] else "False", + 'coverage': coverage} # Testing database sessions if 'session_type' in args: confopts['sesstype'] = args['session_type'] @@ -424,6 +430,10 @@ def parse_args(plugins): parser.add_argument('--session-timeout', default=30, type=int, help='Time that sessions are valid for (in minutes, ' + 'default: 30 minutes)') + parser.add_argument('--coverage', default=False, action='store_true', + help=argparse.SUPPRESS) + parser.add_argument('--coverage-branch', default=False, + action='store_true', help=argparse.SUPPRESS) lms = [] azs = [] diff --git a/ipsilon/ipsilon b/ipsilon/ipsilon index 1e775a2..8fa2bf7 100755 --- a/ipsilon/ipsilon +++ b/ipsilon/ipsilon @@ -15,6 +15,12 @@ import os import atexit import string import cherrypy + +try: + import coverage +except ImportError: # pragma: no cover + coverage = None + from ipsilon import find_config from ipsilon.util.data import AdminStore from ipsilon.util import page @@ -41,6 +47,19 @@ cherrypy.lib.sessions.SqlSession = ipsilon.util.sessions.SqlSession cherrypy.lib.sessions.EtcdSession = ipsilon.util.sessions.EtcdSession cherrypy.config.update(cfgfile) +# If requested, let's capture coverage data +if cherrypy.config.get('coverage.enabled', False): + if not coverage: # pragma: no cover + raise Exception('Coverage.py was unavailable') + + cov = coverage.Coverage( + data_file='coverage', + data_suffix=True, + auto_data=True, + branch=cherrypy.config.get('coverage.branch', False)) + cov.start() + atexit.register(cov.stop) + # Force cherrypy logging to work. Note that this ignores the config-file # setting. cherrypy.log.screen = True diff --git a/templates/install/ipsilon.conf b/templates/install/ipsilon.conf index 6388619..211403f 100644 --- a/templates/install/ipsilon.conf +++ b/templates/install/ipsilon.conf @@ -21,3 +21,5 @@ tools.sessions.path = "${instanceurl}" tools.sessions.timeout = ${session_timeout} tools.sessions.httponly = ${secure} tools.sessions.secure = ${secure} + +${coverage} diff --git a/tests/helpers/common.py b/tests/helpers/common.py index e366db2..20308e3 100755 --- a/tests/helpers/common.py +++ b/tests/helpers/common.py @@ -9,6 +9,7 @@ import pwd import shutil import signal import random +import time from string import Template import subprocess @@ -87,6 +88,8 @@ class IpsilonTestBase(object): self.print_cases = False self.stdout = None self.stderr = None + self.coverage = False + self.coverage_branch = False def platform_supported(self): """This return whether the current platform supports this test. @@ -164,6 +167,10 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir, newconf.add_section('arguments') for k in args_opts: newconf.set('arguments', k, args_opts[k]) + if self.coverage: + newconf.set('arguments', 'coverage', True) + if self.coverage_branch: + newconf.set('arguments', 'coverage_branch', True) profile = io.BytesIO() newconf.write(profile) @@ -246,7 +253,13 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir, cmd = [os.path.join(self.rootdir, 'ipsilon/install/ipsilon-server-install'), '--config-profile=%s' % profile] - subprocess.check_call(cmd, env=env, + if self.coverage: + env['COVERAGE_FILE'] = 'installer.coverage' + covcmd = ['coverage', 'run'] + if self.coverage_branch: + covcmd.append('--branch') + cmd = covcmd + cmd + subprocess.check_call(cmd, env=env, cwd=self.testdir, stdout=self.stdout, stderr=self.stderr) os.symlink(os.path.join(self.rootdir, 'ipsilon'), os.path.join(self.testdir, 'lib', name, 'ipsilon')) @@ -435,9 +448,22 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir, if ksetup.returncode != 0: raise ValueError('kinit %s failed' % self.testuser) + def wait_process(self, p): + start = int(time.time()) + while True: + if int(time.time()) - start > 5: + print('Process %s did not terminate in time, killing' % p.pid) + p.kill() + return + if p.poll() is not None: + return + time.sleep(0.1) + def wait(self): for p in self.processes: os.killpg(p.pid, signal.SIGTERM) + for p in self.processes: + self.wait_process(p) def setup_servers(self, env=None): raise NotImplementedError() diff --git a/tests/tests.py b/tests/tests.py index e5a688f..5986d23 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -9,6 +9,7 @@ import pkg_resources # pylint: disable=unused-import import argparse from ipsilon.util import plugin +import glob import os import sys import subprocess @@ -43,6 +44,13 @@ def parse_args(): help='Test results header') parser.add_argument('--path', default='%s/testdir' % os.getcwd(), help="Directory in which tests are run") + parser.add_argument('--coverage', default=False, action='store_true', + help="Whether or not to collect coverage reports") + parser.add_argument('--coverage-branch', default=False, + action='store_true', + help='Whether to do branch coverage') + parser.add_argument('--coverage-min', default=0, + help='Minimum coverage % to consider success') parser.add_argument('--fail-on-first-error', '-x', action='store_true', help='Abort test run on first test failure') parser.add_argument('--test', action='append', default=None, @@ -105,6 +113,8 @@ def run_test(testname, test, args): devnull = open(os.devnull, 'w') test.stdout = devnull test.stderr = devnull + test.coverage = args['coverage'] + test.coverage_branch = args['coverage_branch'] if args['verbose'] >= VERBOSE_SHOWCASES: test.print_cases = True @@ -158,6 +168,22 @@ def result_is_fail(result): def main(): args = parse_args() + if args['coverage_min'] or args['coverage_branch']: + args['coverage_min'] = int(args['coverage_min']) + args['coverage'] = True + + if args['coverage']: + import coverage + cov = coverage.Coverage( + data_file=os.path.join(args['path'], 'tests.coverage'), + data_suffix=False, + auto_data=False, + branch=args['coverage_branch'] + ) + cov.start() + + coverage_files = [os.path.join(args['path'], 'tests.coverage')] + tests = get_tests() if args['list_tests']: for testname in tests.keys(): @@ -179,6 +205,7 @@ def main(): os.makedirs(args['path']) test_results = {} + anyfailures = False for test in args['test']: if args['verbose'] >= VERBOSE_SHOWTESTS: @@ -189,9 +216,34 @@ def main(): if args['verbose'] >= VERBOSE_SHOWTESTS: print(result_to_str(result)) + anyfailures = anyfailures or result_is_fail(result) + if args['fail_on_first_error'] and result_is_fail(result): break + coverage_files.append(os.path.join(args['path'], + test, + 'installer.coverage')) + coverage_files.extend(glob.glob( + '%s/%s/lib/*/coverage.*' % (args['path'], test))) + + if args['coverage']: + cov.stop() + cov.save() + + if args['coverage'] and not anyfailures: + print('Collecting coverage reports...') + + cov = coverage.Coverage( + data_file=os.path.join(args['path'], 'combined.coverage'), + branch=args['coverage_branch']) + cov.combine(data_paths=coverage_files) + cov.save() + cov_perc = cov.report(include='ipsilon/*') + cov.xml_report( + include='ipsilon/*', + outfile=os.path.join(args['path'], 'combined.coverage.xml')) + if not args['no_overview']: print(args['results_header']) for test in test_results: @@ -201,6 +253,12 @@ def main(): for result in test_results.values()): sys.exit(1) + if (not anyfailures and args['coverage_min'] and + cov_perc < args['coverage_min']): + print('Test coverage was %d%%, less than required %d%%' % + (cov_perc, args['coverage_min'])) + sys.exit(1) + if __name__ == '__main__': main() From 5aaaeda23bede79503e11c41a0b2e54cf55383a8 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Nov 20 2017 16:48:03 +0000 Subject: [PATCH 2/2] Run coverage tests with min coverage for Fedora27 containertest Signed-off-by: Patrick Uiterwijk --- diff --git a/Makefile b/Makefile index 536361a..9faa3b3 100644 --- a/Makefile +++ b/Makefile @@ -170,7 +170,11 @@ containertest-fedora26: container-fedora26 containertest-fedora27: container-fedora27 @echo "Starting Fedora 27 tests ..." - @docker run -v `pwd`:/code -t --rm ipsilon-fedora27 + # This one is special, since it's the latest version, so supposed to support the most features. + # As such, we do coverage testing on this one. + @docker run -v `pwd`:/code -t --rm --entrypoint /usr/bin/make ipsilon-fedora27 testdeps lp-test unittests + # This "coverage-min" number should be increased everytime the coverage increases to make sure we don't regress. + @docker run -v `pwd`:/code -t --rm --entrypoint /usr/bin/bash ipsilon-fedora27 ./runtests --path=$(TESTDIR) --coverage --coverage-min=68 @echo "Fedora 27 passed" containertest-lint: container-centos7 diff --git a/tests/containers/Dockerfile-fedora b/tests/containers/Dockerfile-fedora index c519795..f3de0e6 100644 --- a/tests/containers/Dockerfile-fedora +++ b/tests/containers/Dockerfile-fedora @@ -1 +1 @@ -RUN yum install -y etcd python2-python-etcd dbus-python python2-ipalib +RUN yum install -y etcd python2-python-etcd dbus-python python2-ipalib python2-coverage