From 47fdd0e47f23eb27900960ac900f0a0bbc32cb1f Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: May 22 2017 07:01:34 +0000 Subject: [PATCH 1/3] code cleanup --- diff --git a/modularity/module_deps_differ.py b/modularity/module_deps_differ.py index 2cc8317..7006785 100755 --- a/modularity/module_deps_differ.py +++ b/modularity/module_deps_differ.py @@ -2,7 +2,6 @@ from pdc_client import PDCClient import dnf import os -import sys import re @@ -57,30 +56,6 @@ class ModuleDepsDiffer(object): for module in j: self.module_id_to_name[module["koji_tag"]] = "{}:{}".format(module["variant_id"], module["variant_version"]) - def get_repo_urls(self): - """ this doesn't work anymore since the repos are no longer present in koji """ - for repo in SUPPORTED_MODULES: - module_name = repo - stream = STREAM - repo_url_template = "https://kojipkgs.fedoraproject.org/repos/{}" - j = self.p['unreleasedvariants']( - variant_name=module_name, - variant_type="module", - variant_version=stream, - active=True - ) - if not j["results"]: - # fallback - j = self.p['unreleasedvariants']( - variant_name=module_name, - variant_type="module", - variant_version=FALLBACK_STREAM, - active=True - ) - latest = max(j["results"], key=lambda x:x['variant_release']) - repo_url = repo_url_template.format(latest["koji_tag"]) - self.repos[repo] = str(repo_url + '/latest/' + ARCH) - @staticmethod def get_base_from_repo(reponame, repourl): base = dnf.Base() @@ -93,13 +68,6 @@ class ModuleDepsDiffer(object): return base def _init_repo_bases(self): - """ initialize repo objects for dnf to kick off queries """ - self.get_repo_urls() - for reponame, repourl in self.repos.items(): - base = ModuleDepsDiffer.get_base_from_repo(reponame, repourl) - self.repo_bases[reponame] = base - - def _init_repo_bases2(self): """ version 1 is no longer usable since koji no longer creates repos for modules this version iterates over repos present in @@ -110,7 +78,9 @@ class ModuleDepsDiffer(object): initialize repo objects for dnf to kick off queries """ - print("Getting module information from mbs cache. Please run mbs-build local with dependency modules you are interested in before running this script.") + print("Getting module information from mbs cache." + + " Please run mbs-build local with dependency modules" + + " you are interested in before running this script.") cache_path = os.path.expanduser("~/modulebuild/cache/koji_tags") for module_name in os.listdir(cache_path): if module_name != "f26-modularity": # bootstrap contains almost everything, so let's ignore it @@ -119,31 +89,22 @@ class ModuleDepsDiffer(object): base = ModuleDepsDiffer.get_base_from_repo(reponame, repourl) self.repo_bases[reponame] = base - def get_repo_provides(self): - for reponame, base in self.repo_bases.items(): - allpkgs = base.sack.query().filter() - provides_list = set() - for pkg in allpkgs: - for cap in getattr(pkg, 'provides'): - provides_list.add(str(cap)) - self.repo_provides[reponame] = provides_list - def get_package_requires(self): # runtime base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26) pkgs = base.sack.query().filter(name=self.package, arch=['noarch', ARCH], latest=True) ModuleDepsDiffer.package_unique(pkgs) requires = getattr(pkgs[0], 'requires') - for p in requires: - self.runtime_req.add(str(p)) + for q in requires: + self.runtime_req.add(str(q)) # build base = ModuleDepsDiffer.get_base_from_repo('f26-source', REPO_F26_SOURCE) pkgs = base.sack.query().filter(name=pkgs[0].source_name) ModuleDepsDiffer.package_unique(pkgs) requires = getattr(pkgs[0], 'requires') - for p in requires: - self.build_req.add(str(p)) + for q in requires: + self.build_req.add(str(q)) # Igor suggests to do complete dependency solving here: # base.install('package'), installroot=, make sure the right repos are enabled @@ -179,14 +140,12 @@ class ModuleDepsDiffer(object): for cap in caps: pkg = q.filter(provides__glob=[cap], arch=['noarch', ARCH], latest=True) if pkg: - stripped = re.sub(r'-[^-]*-[^-]*$', '', str(pkg.result[0].source_name)) - result.add(stripped) + result.add(str(pkg.result[0].source_name)) else: pkg = q.filter(file__glob=cap) if len(pkg) == 0: raise ValueError('No package provides capability \'' + cap + '\'') - stripped = re.sub(r'-[^-]*-[^-]*$', '', str(pkg.result[0].source_name)) - result.add(stripped) + result.add(str(pkg.result[0].source_name)) return result def _dump_caps_result(self, items): @@ -196,8 +155,8 @@ class ModuleDepsDiffer(object): if repo == 'result': print("Capabilities which aren't provided by any module:") else: - print("Components which are part of " + \ - self.get_module_name(repo) + \ + print("Components which are part of " + + self.get_module_name(repo) + " module:") for cap in sorted(caps): print(cap) @@ -219,8 +178,8 @@ class ModuleDepsDiffer(object): if repo == 'result': print("Components which aren't provided by any module:") else: - print("Components which are part of " + \ - self.get_module_name(repo) + \ + print("Components which are part of " + + self.get_module_name(repo) + " module:") for cap in sorted(ModuleDepsDiffer.whatprovides(caps)): print(cap) @@ -236,7 +195,7 @@ class ModuleDepsDiffer(object): self._dump_pkgs_result(self.runtime_caps_classified.items()) def run(self): - self._init_repo_bases2() + self._init_repo_bases() self.get_package_requires() # self.get_repo_provides() self.obtain_module_names() From 8706a8be0bb6cc8d2f0555708ba8b50df9ffd76a Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: May 22 2017 07:59:16 +0000 Subject: [PATCH 2/3] Require python3 and enable multiple package resolution in one run --- diff --git a/diff_module_deps.py b/diff_module_deps.py index b3ea5fa..fc48aea 100755 --- a/diff_module_deps.py +++ b/diff_module_deps.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import sys from modularity.cli import CliModuleDepsDifferHelper if __name__ == "__main__": - sys.exit(CliModuleDepsDifferHelper.run()) \ No newline at end of file + sys.exit(CliModuleDepsDifferHelper.run()) diff --git a/modularity/cli.py b/modularity/cli.py index 69db9b6..b4213d6 100644 --- a/modularity/cli.py +++ b/modularity/cli.py @@ -6,7 +6,8 @@ import argparse from modularity.module_generator import ModuleGenerator from modularity.oc_template import OpenShiftTemplateGenerator -from modularity.module_deps_differ import ModuleDepsDiffer +from .module_deps_differ import ModuleDepsDiffer + class CLI(object): """ Class for processing data from commandline """ @@ -50,6 +51,7 @@ class CliHelper(object): # print(e) # sys.exit(1) + class CLIRpm2Module(object): """ Class for processing data from commandline """ @@ -59,6 +61,7 @@ class CLIRpm2Module(object): parser.add_argument( "pkgs", metavar='PKGS', + nargs='+', help="Specify list of packages for module.", ) return parser @@ -86,7 +89,8 @@ class CliModuleDepsDiffer(object): parser.add_argument( "pkgs", metavar='PKGS', - help="Name of package for dependency resolution" + nargs='+', + help="Names of packages for dependency resolution" ) parser.add_argument( "--resolve", @@ -108,7 +112,7 @@ class CliModuleDepsDifferHelper(object): def run(): try: cli = CliModuleDepsDiffer(sys.argv[1:]) - mdd = ModuleDepsDiffer(cli) + mdd = ModuleDepsDiffer(cli.args.pkgs) mdd.run() if cli.resolve: mdd.dump_pkgs_result() @@ -124,7 +128,7 @@ class CliRpm2ModuleHelper(object): def run(): try: cli = CLIRpm2Module(sys.argv[1:]) - mg = ModuleGenerator(cli) + mg = ModuleGenerator(cli.args.pkgs) mg.run() except KeyboardInterrupt: @@ -132,4 +136,3 @@ class CliRpm2ModuleHelper(object): # except Exception as e: # print(e) # sys.exit(1) - diff --git a/modularity/module_deps_differ.py b/modularity/module_deps_differ.py index 7006785..ea109df 100755 --- a/modularity/module_deps_differ.py +++ b/modularity/module_deps_differ.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from pdc_client import PDCClient import dnf import os -import re FALLBACK_STREAM = 'master' @@ -28,12 +27,12 @@ def get_pdc_client(): class ModuleDepsDiffer(object): - def __init__(self, args): + def __init__(self, pkgs): self.p = get_pdc_client() self.repos = dict() self.repo_provides = dict() # TODO: get this from args - self.package = args.pkgs + self.packages = list(pkgs) self.build_req = set() self.runtime_req = set() self.build_caps_classified = dict() @@ -83,6 +82,9 @@ class ModuleDepsDiffer(object): " you are interested in before running this script.") cache_path = os.path.expanduser("~/modulebuild/cache/koji_tags") for module_name in os.listdir(cache_path): + # for now ignore all what is not in f26 stream, remove when situation changes + if not self.module_id_to_name[module_name].endswith(':f26'): + continue if module_name != "f26-modularity": # bootstrap contains almost everything, so let's ignore it self.repos[module_name] = "file://{}".format(os.path.join(cache_path, module_name)) for reponame, repourl in self.repos.items(): @@ -91,20 +93,28 @@ class ModuleDepsDiffer(object): def get_package_requires(self): # runtime + pkgs = [] base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26) - pkgs = base.sack.query().filter(name=self.package, arch=['noarch', ARCH], latest=True) - ModuleDepsDiffer.package_unique(pkgs) - requires = getattr(pkgs[0], 'requires') - for q in requires: - self.runtime_req.add(str(q)) + for pkg in self.packages: + filter_result = base.sack.query().filter(name=pkg, arch=['noarch', ARCH], latest=True) + ModuleDepsDiffer.package_unique(filter_result) + pkgs.append(filter_result[0]) + for pkg in pkgs: + requires = getattr(pkg, 'requires') + for q in requires: + self.runtime_req.add(str(q)) # build + source_pkgs = [] base = ModuleDepsDiffer.get_base_from_repo('f26-source', REPO_F26_SOURCE) - pkgs = base.sack.query().filter(name=pkgs[0].source_name) - ModuleDepsDiffer.package_unique(pkgs) - requires = getattr(pkgs[0], 'requires') - for q in requires: - self.build_req.add(str(q)) + for pkg in pkgs: + filter_result = base.sack.query().filter(name=pkg.source_name) + ModuleDepsDiffer.package_unique(filter_result) + source_pkgs.append(filter_result[0]) + for pkg in source_pkgs: + requires = getattr(pkg, 'requires') + for q in requires: + self.build_req.add(str(q)) # Igor suggests to do complete dependency solving here: # base.install('package'), installroot=, make sure the right repos are enabled @@ -195,9 +205,8 @@ class ModuleDepsDiffer(object): self._dump_pkgs_result(self.runtime_caps_classified.items()) def run(self): + self.obtain_module_names() self._init_repo_bases() self.get_package_requires() - # self.get_repo_provides() - self.obtain_module_names() self.classify_caps(self.build_caps_classified, self.build_req) self.classify_caps(self.runtime_caps_classified, self.runtime_req) diff --git a/modularity/module_generator.py b/modularity/module_generator.py index bbe2c6d..790d82f 100644 --- a/modularity/module_generator.py +++ b/modularity/module_generator.py @@ -1,17 +1,17 @@ import modulemd import dnf -from module_deps_differ import ModuleDepsDiffer +from .module_deps_differ import ModuleDepsDiffer class ModuleGenerator(object): - def __init__(self, args): - self.args = args + def __init__(self, pkgs): + self.pkgs = pkgs self.pkg = None self.mmd = modulemd.ModuleMetadata() self.build_deps = set() self.run_deps = set() - self.differ = ModuleDepsDiffer(args) + self.differ = ModuleDepsDiffer(pkgs) def _save_module_md(self): """ @@ -19,7 +19,11 @@ class ModuleGenerator(object): based on argument name :return: """ - file_name = self.args.pkgs + '.yaml' + + if len(self.pkgs) == 1: + file_name = self.pkgs[0] + '.yaml' + else: + file_name = "modulemd-output.yaml" self.mmd.dump(file_name) print('Modulemd file is generated here ./%s' % file_name) return True @@ -31,14 +35,15 @@ class ModuleGenerator(object): :return: """ - self.mmd.summary = str(self.pkg.summary) - self.mmd.description = str(self.pkg.description) + if len(self.pkgs) == 1: + self.mmd.summary = str(self.pkg.summary) + self.mmd.description = str(self.pkg.description) - # Default license for the module metadata, same as default Fedora - # content license. - self.mmd.add_module_license("MIT") + # Default license for the module metadata, same as default Fedora + # content license. + self.mmd.add_module_license("MIT") - self.mmd.add_content_license(str(self.pkg.license)) + self.mmd.add_content_license(str(self.pkg.license)) for pkg in self.build_deps.intersection(self.run_deps): self.mmd.components.add_rpm(pkg, "Build and runtime dependency.") @@ -54,13 +59,15 @@ class ModuleGenerator(object): continue if caps: # TODO: use specific stream? - self.mmd.add_buildrequires(mod, 'master') + name, stream = self.differ.get_module_name(mod).split(':') + self.mmd.add_buildrequires(name, stream) for mod, caps in self.differ.runtime_caps_classified.items(): if mod == 'result': continue if caps: - self.mmd.add_requires(mod, 'master') + name, stream = self.differ.get_module_name(mod).split(':') + self.mmd.add_requires(name, stream) def _get_pkg_info(self): """ @@ -71,7 +78,7 @@ class ModuleGenerator(object): b.read_all_repos() b.fill_sack() - q = b.sack.query().filter(name=self.args.pkgs, reponame='fedora', latest=True) + q = b.sack.query().filter(name=self.pkgs, reponame='fedora', latest=True) if len(q) > 1: raise ValueError('Name of package is not unique') @@ -89,7 +96,8 @@ class ModuleGenerator(object): self.run_deps = ModuleDepsDiffer.whatprovides(self.differ.runtime_caps_classified['result']) def run(self): - self._get_pkg_info() + if len(self.pkgs) == 1: + self._get_pkg_info() self._get_dependencies() self._update_module_md() self._save_module_md() From 1493b59cd32456ae97b07aa6abef1adf2552a61d Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: May 22 2017 08:22:24 +0000 Subject: [PATCH 3/3] Update README file --- diff --git a/README.md b/README.md index 1815ac4..1706fc5 100644 --- a/README.md +++ b/README.md @@ -32,16 +32,9 @@ In order to run tests, run command: `py.test-2.7 tests/` -### Base Run Time dependency solver - * `./brt_dep_solver.sh ` - produce files with dependencies +### Modulemd creation -For verbose output set variable VERBOSE to true: - - # VERBOSE=true ./brt_dep_solver.sh - -For debugging you might want to keep all files generated by dependecy solver -(not just the ones with resuls). -Set variable CLEAN to false to keep all these files. - - # VERBOSE=false ./brt_dep_solver.sh +./rpm2modulemd.py script creates modulemd file from package names. Output is written in modulemd-output.yaml +file (multiple packages as input) or in file named after package name (single package as input). +Please make sure you have latest module-build-service and dnf installed before running.