From eb70db8eec2107f6a8fadb0881e09b165896bd29 Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: Apr 25 2017 20:33:40 +0000 Subject: [PATCH 1/5] Improve ./brt_dep_solver.sh to diff_module_deps.py --- diff --git a/diff_module_deps.py b/diff_module_deps.py new file mode 100755 index 0000000..b3ea5fa --- /dev/null +++ b/diff_module_deps.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import sys +from modularity.cli import CliModuleDepsDifferHelper + +if __name__ == "__main__": + sys.exit(CliModuleDepsDifferHelper.run()) \ No newline at end of file diff --git a/modularity/cli.py b/modularity/cli.py index d46cc5e..1cb93c8 100644 --- a/modularity/cli.py +++ b/modularity/cli.py @@ -6,7 +6,7 @@ import argparse from modularity.module_generator import ModuleGenerator from modularity.oc_template import OpenShiftTemplateGenerator - +from modularity.module_deps_differ import ModuleDepsDiffer class CLI(object): """ Class for processing data from commandline """ @@ -74,6 +74,50 @@ class CLIRpm2Module(object): return object.__getattribute__(self, name) +class CliModuleDepsDiffer(object): + + def __init__(self, args=None): + self.parser = CliModuleDepsDiffer.build_parser() + self.args = self.parser.parse_args(args) + + @staticmethod + def build_parser(): + parser = argparse.ArgumentParser(description='k;') + parser.add_argument( + "pkg", + metavar='PKG', + help="Name of package for dependency resolution" + ) + parser.add_argument( + "--resolve", + action='store_true', + default=False + ) + + return parser + + def __getattr__(self, name): + try: + return getattr(self.args, name) + except AttributeError: + return object.__getattribute__(self, name) + + +class CliModuleDepsDifferHelper(object): + @staticmethod + def run(): + try: + cli = CliModuleDepsDiffer(sys.argv[1:]) + mdd = ModuleDepsDiffer(cli) + mdd.run() + if cli.resolve: + mdd.dump_pkgs_result() + else: + mdd.dump_caps_result() + except KeyboardInterrupt: + print('\nInterrupted by user') + + class CliRpm2ModuleHelper(object): @staticmethod diff --git a/modularity/module_deps_differ.py b/modularity/module_deps_differ.py new file mode 100755 index 0000000..a359899 --- /dev/null +++ b/modularity/module_deps_differ.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +from pdc_client import PDCClient +import dnf +import sys +import re + + +SUPPORTED_MODULES = ['perl', 'base-runtime', 'common-build-dependencies-bootstrap', 'common-build-dependencies'] +STREAM = 'master' +REPO_F26 = "http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/x86_64/os/" +REPO_F26_SOURCE = "http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/source/tree/" +ARCH = 'x86_64' + +# TODO: add ability to solve more packages in one run +# TODO: solve architecture! - as option? + + +class ModuleDepsDiffer(object): + + def __init__(self, args): + self.repos = dict() + self.repo_provides = dict() + # TODO: get this from args + self.package = args.pkg + self.build_req = set() + self.runtime_req = set() + self.build_caps_classified = dict() + self.build_caps_classified['result'] = set() + self.runtime_caps_classified = dict() + self.runtime_caps_classified['result'] = set() + + def get_repo_urls(self): + for repo in SUPPORTED_MODULES: + p = PDCClient('https://pdc.fedoraproject.org/rest_api/v1/', develop=True, ssl_verify=True) + + module_name = repo + stream = STREAM + repo_url_template = "https://kojipkgs.fedoraproject.org/repos/{}" + j = p['unreleasedvariants']( + variant_name=module_name, + variant_type="module", + variant_version=stream, + active=True + ) + # TODO: get latest + latest = max(j["results"], key=lambda x:x['variant_release']) + repo_url = repo_url_template.format(latest["koji_tag"]) + self.repos[repo] = str(repo_url + '/latest/' + ARCH) + + @staticmethod + def get_base_from_repo(reponame, repourl): + base = dnf.Base() + repo = dnf.repo.Repo(reponame, base.conf) + repo.baseurl = repourl + repo.load() + repo.enable() + base.repos.add(repo) + base.fill_sack(load_available_repos=True, load_system_repo=False) + return base + + def get_repo_provides(self): + self.get_repo_urls() + for reponame, repourl in self.repos.items(): + base = ModuleDepsDiffer.get_base_from_repo(reponame, repourl) + allpkgs = base.sack.query().filter() + provides_list = set() + for pkg in allpkgs: + for cap in getattr(pkg, 'provides'): + provides_list.add(str(cap)) + + self.repo_provides[reponame] = provides_list + + def get_package_requires(self): + # runtime + base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26) + pkgs = base.sack.query().filter(name=self.package) + ModuleDepsDiffer.package_unique(pkgs) + requires = getattr(pkgs[0], 'requires') + for p in requires: + self.runtime_req.add(str(p)) + + # build + base = ModuleDepsDiffer.get_base_from_repo('f26-source', REPO_F26_SOURCE) + pkgs = base.sack.query().filter(name=pkgs[0].source_name) + ModuleDepsDiffer.package_unique(pkgs) + requires = getattr(pkgs[0], 'requires') + for p in requires: + self.build_req.add(str(p)) + + def classify_caps(self, caps_classified, requirements): + for repo_name, repo_provides in self.repo_provides.items(): + caps_classified[repo_name] = set() + + for cap in requirements: + if " = " not in cap: + r = re.compile(str('^' + re.escape(cap) + '*')) + else: + r = re.compile(re.escape(cap)) + found = False + for repo_name, repo_provides in self.repo_provides.items(): + if list(filter(r.match, repo_provides)): + found = True + caps_classified[repo_name].add(cap) + if not found: + caps_classified['result'].add(cap) + + @staticmethod + def package_unique(result): + if len(result) > 1: + raise ValueError('Name of package is not unique') + if len(result) == 0: + raise ValueError('No package found in repo') + + @staticmethod + def whatprovides(caps): + result = set() + base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26) + q = base.sack.query() + for cap in caps: + pkg = q.filter(provides__glob=[cap], latest=True) + if pkg: + stripped = re.sub(r'-[^-]*-[^-]*$', '', str(pkg.result[0])) + result.add(stripped) + else: + q = q.filter(file__glob=cap) + if len(q) == 0: + raise ValueError('No package provides capability \'' + cap + '\'') + stripped = re.sub(r'-[^-]*-[^-]*$', '', str(q.result[0])) + result.add(stripped) + return result + + @staticmethod + def _dump_caps_result(items): + for repo, caps in items: + if not caps: + continue + if repo == 'result': + print("Capabilities which aren't provided by any module:") + else: + print("Capabilities which are part of " + repo + " module:") + for cap in sorted(caps): + print(cap) + print('') + + def dump_caps_result(self): + print('BUILD requirements:') + print('-------------------') + ModuleDepsDiffer._dump_caps_result(self.build_caps_classified.items()) + print('\n') + print('RUNTIME requirements:') + print('---------------------') + ModuleDepsDiffer._dump_caps_result(self.runtime_caps_classified.items()) + + @staticmethod + def _dump_pkgs_result(items): + for repo, caps in items: + if not caps: + continue + if repo == 'result': + print("Components which aren't provided by any module:") + else: + print("Components which are part of " + repo + " module:") + for cap in sorted(ModuleDepsDiffer.whatprovides(caps)): + print(cap) + print('') + + def dump_pkgs_result(self): + print('BUILD dependencies:') + print('-------------------') + self._dump_pkgs_result(self.build_caps_classified.items()) + print('\n') + print('RUNTIME dependencies:') + print('---------------------') + self._dump_pkgs_result(self.runtime_caps_classified.items()) + + def run(self): + self.get_repo_provides() + self.get_package_requires() + self.classify_caps(self.build_caps_classified, self.build_req) + self.classify_caps(self.runtime_caps_classified, self.runtime_req) From 5c09b8462bc670d6412943fca42c4a4adf441703 Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: Apr 25 2017 20:38:59 +0000 Subject: [PATCH 2/5] Remove unneeded ./brt_dep_solver.sh script --- diff --git a/brt_dep_solver.sh b/brt_dep_solver.sh deleted file mode 100755 index 22c76de..0000000 --- a/brt_dep_solver.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -# set -ex -pkg=${1} - -CLEAN=${CLEAN:-true} -VERBOSE=${VERBOSE:-false} -REPO_F26="http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/x86_64/os/" -REPO_F26_SOURCE="http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/source/tree/" - -function verbose_echo { - [ "$VERBOSE" != "true" ] && return - - echo "$@" -} - -# TODO: add other known dependency modules -function get-baseruntime-caps { - - provided_caps=brt-provides.txt - - verbose_echo -n "Getting capabilities provided by base-runtime packages... " - dnf repoquery --disablerepo='*' --repofrompath brt,https://kojipkgs.stg.fedoraproject.org/compose/branched/jkaluza/latest-Boltron-26/compose/base-runtime/x86_64/os/ --enablerepo=brt --provides -q > ${provided_caps} && verbose_echo "DONE" - -} - -function cleanup { - - cd .. - ${CLEAN} && rm ${DIR}/* && rmdir ${DIR} -} - -function resolve-deps { - if [[ ${1} == "runtime" ]]; then - runtime=true - elif [[ ${1} == "build" ]]; then - runtime=false - else - echo "resolve-deps: Invalid operation ${1}" - exit -1 - fi - - if [ "${runtime}" = "true" ]; then - DIR=${pkg}-runtime-res-files - else - DIR=${pkg}-build-res-files - fi - - mkdir ${DIR} - cd ${DIR} - - get-baseruntime-caps - - verbose_echo -n "Getting ${1} capabilities required by ${pkg}... " - # TODO: use remote path for fedora repository - SRPM_FLAG= - [ "${runtime}" = "true" ] || SRPM_FLAG=--srpm - dnf repoquery --requires $SRPM_FLAG -q \ - --disablerepo='*' \ - --repofrompath=f26,${REPO_F26} \ - --repofrompath=f26-source,${REPO_F26_SOURCE} \ - --enablerepo=f26 \ - ${pkg} > ${pkg}-caps.txt && verbose_echo "DONE" - -# [ -s ${pkg}-caps.txt ] || cleanup && return - if [ ! -s ${pkg}-caps.txt ]; then - cleanup - verbose_echo "No ${1} dependencies found." - return - fi - while read cap; do - if [[ $cap != *" = "* ]]; then - grep -wq "^${cap}*" ${provided_caps} || echo ${cap} >> ${pkg}-filtered-caps.txt - else - grep -wq "${cap}" ${provided_caps} || echo ${cap} >> ${pkg}-filtered-caps.txt - fi - done < ${pkg}-caps.txt - - if [ ! -s ${pkg}-filtered-caps.txt ]; then - cleanup - verbose_echo "No ${1} dependencies found after diff with baseruntime." - return - fi - while read cap; do - dnf repoquery -q --latest-limit=1 --srpm \ - --disablerepo='*' \ - --repofrompath f26,${REPO_F26} \ - --repofrompath=f26-source,${REPO_F26_SOURCE} \ - --enablerepo=f26 \ - --whatprovides "${cap}" >> cap2rpm.txt - done < ${pkg}-filtered-caps.txt - - if [ "$runtime" = "true" ]; then - sed -e "s/-[^-]*-[^-]*$//" cap2rpm.txt | sort -u > ../${pkg}-runtime-deps.txt - else - sed -e "s/-[^-]*-[^-]*$//" cap2rpm.txt | sort -u > ../${pkg}-build-deps.txt - fi - - cleanup -} - -resolve-deps "build" -resolve-deps "runtime" From 20995433481b395ce75e9ca9dcfcb24226d68ed4 Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: Apr 26 2017 14:04:57 +0000 Subject: [PATCH 3/5] Use ModuleDepsDiffer when creating modulemd --- diff --git a/modularity/cli.py b/modularity/cli.py index 1cb93c8..69db9b6 100644 --- a/modularity/cli.py +++ b/modularity/cli.py @@ -84,8 +84,8 @@ class CliModuleDepsDiffer(object): def build_parser(): parser = argparse.ArgumentParser(description='k;') parser.add_argument( - "pkg", - metavar='PKG', + "pkgs", + metavar='PKGS', help="Name of package for dependency resolution" ) parser.add_argument( diff --git a/modularity/module_deps_differ.py b/modularity/module_deps_differ.py index a359899..2969144 100755 --- a/modularity/module_deps_differ.py +++ b/modularity/module_deps_differ.py @@ -21,7 +21,7 @@ class ModuleDepsDiffer(object): self.repos = dict() self.repo_provides = dict() # TODO: get this from args - self.package = args.pkg + self.package = args.pkgs self.build_req = set() self.runtime_req = set() self.build_caps_classified = dict() @@ -119,13 +119,13 @@ class ModuleDepsDiffer(object): for cap in caps: pkg = q.filter(provides__glob=[cap], latest=True) if pkg: - stripped = re.sub(r'-[^-]*-[^-]*$', '', str(pkg.result[0])) + stripped = re.sub(r'-[^-]*-[^-]*$', '', str(pkg.result[0].source_name)) result.add(stripped) else: q = q.filter(file__glob=cap) if len(q) == 0: raise ValueError('No package provides capability \'' + cap + '\'') - stripped = re.sub(r'-[^-]*-[^-]*$', '', str(q.result[0])) + stripped = re.sub(r'-[^-]*-[^-]*$', '', str(q.result[0].source_name)) result.add(stripped) return result diff --git a/modularity/module_generator.py b/modularity/module_generator.py index 0e0a966..2c1b630 100644 --- a/modularity/module_generator.py +++ b/modularity/module_generator.py @@ -1,7 +1,6 @@ -import os -import subprocess import modulemd import dnf +from module_deps_differ import ModuleDepsDiffer class ModuleGenerator(object): @@ -12,6 +11,7 @@ class ModuleGenerator(object): self.mmd = modulemd.ModuleMetadata() self.build_deps = set() self.run_deps = set() + self.differ = ModuleDepsDiffer(args) def _save_module_md(self): """ @@ -71,18 +71,9 @@ class ModuleGenerator(object): Function gets build and runtime dependencies of package :return: """ - subprocess.call(['/bin/bash', './brt_dep_solver.sh', self.args.pkgs]) - - self.build_deps_file = self.args.pkgs + '-build-deps.txt' - self.run_deps_file = self.args.pkgs + '-runtime-deps.txt' - - if os.path.exists(self.build_deps_file): - with open(self.build_deps_file, 'r') as f: - self.build_deps = set([l.strip('\n') for l in f.readlines()]) - - if os.path.exists(self.run_deps_file): - with open(self.run_deps_file, 'r') as f: - self.run_deps = set([l.strip('\n') for l in f.readlines()]) + self.differ.run() + self.build_deps = ModuleDepsDiffer.whatprovides(self.differ.build_caps_classified['result']) + self.run_deps = ModuleDepsDiffer.whatprovides(self.differ.runtime_caps_classified['result']) def run(self): self._get_pkg_info() From 014f2fecf87a07f38f52c4129bd56cd8bd450ff2 Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: Apr 27 2017 07:10:24 +0000 Subject: [PATCH 4/5] Fill required dependency modules in modulemd --- diff --git a/modularity/module_generator.py b/modularity/module_generator.py index 2c1b630..bbe2c6d 100644 --- a/modularity/module_generator.py +++ b/modularity/module_generator.py @@ -49,6 +49,19 @@ class ModuleGenerator(object): for pkg in (self.run_deps - self.build_deps): self.mmd.components.add_rpm(pkg, "Runtime dependency.") + for mod, caps in self.differ.build_caps_classified.items(): + if mod == 'result': + continue + if caps: + # TODO: use specific stream? + self.mmd.add_buildrequires(mod, 'master') + + for mod, caps in self.differ.runtime_caps_classified.items(): + if mod == 'result': + continue + if caps: + self.mmd.add_requires(mod, 'master') + def _get_pkg_info(self): """ Function loads package from dnf From 168d1aba838996dbb815c77a9be9b8c079f01edd Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: Apr 27 2017 11:47:13 +0000 Subject: [PATCH 5/5] Fix issue with more packages in query result --- diff --git a/modularity/module_deps_differ.py b/modularity/module_deps_differ.py index 2969144..a95dc36 100755 --- a/modularity/module_deps_differ.py +++ b/modularity/module_deps_differ.py @@ -73,7 +73,7 @@ class ModuleDepsDiffer(object): def get_package_requires(self): # runtime base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26) - pkgs = base.sack.query().filter(name=self.package) + pkgs = base.sack.query().filter(name=self.package, arch=['noarch', ARCH], latest=True) ModuleDepsDiffer.package_unique(pkgs) requires = getattr(pkgs[0], 'requires') for p in requires: @@ -107,7 +107,7 @@ class ModuleDepsDiffer(object): @staticmethod def package_unique(result): if len(result) > 1: - raise ValueError('Name of package is not unique') + raise ValueError('Name of package is not unique: ' + str(result.result)) if len(result) == 0: raise ValueError('No package found in repo') @@ -117,7 +117,7 @@ class ModuleDepsDiffer(object): base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26) q = base.sack.query() for cap in caps: - pkg = q.filter(provides__glob=[cap], latest=True) + pkg = q.filter(provides__glob=[cap], arch=['noarch', ARCH], latest=True) if pkg: stripped = re.sub(r'-[^-]*-[^-]*$', '', str(pkg.result[0].source_name)) result.add(stripped) @@ -174,7 +174,7 @@ class ModuleDepsDiffer(object): self._dump_pkgs_result(self.runtime_caps_classified.items()) def run(self): - self.get_repo_provides() self.get_package_requires() + self.get_repo_provides() self.classify_caps(self.build_caps_classified, self.build_req) self.classify_caps(self.runtime_caps_classified, self.runtime_req)