From e76b37eee65f302f5218d4b18c278136f1cd627b Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Apr 27 2017 13:04:29 +0000 Subject: [PATCH 1/2] use f26 primarily, fallback to master Signed-off-by: Tomas Tomecek --- diff --git a/modularity/module_deps_differ.py b/modularity/module_deps_differ.py index a95dc36..53a2d9a 100755 --- a/modularity/module_deps_differ.py +++ b/modularity/module_deps_differ.py @@ -6,7 +6,8 @@ import re SUPPORTED_MODULES = ['perl', 'base-runtime', 'common-build-dependencies-bootstrap', 'common-build-dependencies'] -STREAM = 'master' +FALLBACK_STREAM = 'master' +STREAM = 'f26' REPO_F26 = "http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/x86_64/os/" REPO_F26_SOURCE = "http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/source/tree/" ARCH = 'x86_64' @@ -15,9 +16,21 @@ ARCH = 'x86_64' # TODO: solve architecture! - as option? +p = None + + +def get_pdc_client(): + """ cache PDCClient instance """ + global p + if p is None: + p = PDCClient('https://pdc.fedoraproject.org/rest_api/v1/', develop=True, ssl_verify=True) + return p + + class ModuleDepsDiffer(object): def __init__(self, args): + self.p = get_pdc_client() self.repos = dict() self.repo_provides = dict() # TODO: get this from args @@ -31,18 +44,23 @@ class ModuleDepsDiffer(object): def get_repo_urls(self): for repo in SUPPORTED_MODULES: - p = PDCClient('https://pdc.fedoraproject.org/rest_api/v1/', develop=True, ssl_verify=True) - module_name = repo stream = STREAM repo_url_template = "https://kojipkgs.fedoraproject.org/repos/{}" - j = p['unreleasedvariants']( + j = self.p['unreleasedvariants']( variant_name=module_name, variant_type="module", variant_version=stream, active=True ) - # TODO: get latest + if not j["results"]: + # fallback + j = self.p['unreleasedvariants']( + variant_name=module_name, + variant_type="module", + variant_version=FALLBACK_STREAM, + active=True + ) latest = max(j["results"], key=lambda x:x['variant_release']) repo_url = repo_url_template.format(latest["koji_tag"]) self.repos[repo] = str(repo_url + '/latest/' + ARCH) From 8f58f1c0a0ce9f4e49533273964dfd8ae04d534b Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Apr 28 2017 13:05:29 +0000 Subject: [PATCH 2/2] resolve requires one by one using dnf Signed-off-by: Tomas Tomecek --- diff --git a/modularity/module_deps_differ.py b/modularity/module_deps_differ.py index 53a2d9a..77fade9 100755 --- a/modularity/module_deps_differ.py +++ b/modularity/module_deps_differ.py @@ -41,6 +41,7 @@ class ModuleDepsDiffer(object): self.build_caps_classified['result'] = set() self.runtime_caps_classified = dict() self.runtime_caps_classified['result'] = set() + self.repo_bases = {} def get_repo_urls(self): for repo in SUPPORTED_MODULES: @@ -76,16 +77,20 @@ class ModuleDepsDiffer(object): base.fill_sack(load_available_repos=True, load_system_repo=False) return base - def get_repo_provides(self): + def _init_repo_bases(self): + """ initialize repo objects for dnf to kick off queries """ self.get_repo_urls() for reponame, repourl in self.repos.items(): base = ModuleDepsDiffer.get_base_from_repo(reponame, repourl) + self.repo_bases[reponame] = base + + def get_repo_provides(self): + for reponame, base in self.repo_bases.items(): allpkgs = base.sack.query().filter() provides_list = set() for pkg in allpkgs: for cap in getattr(pkg, 'provides'): provides_list.add(str(cap)) - self.repo_provides[reponame] = provides_list def get_package_requires(self): @@ -105,21 +110,21 @@ class ModuleDepsDiffer(object): for p in requires: self.build_req.add(str(p)) + # Igor suggests to do complete dependency solving here: + # base.install('package'), installroot=, make sure the right repos are enabled + # base.resolve() + # and then analyze base.transaction.install_set + # weak deps will get resolved correctly with this approach def classify_caps(self, caps_classified, requirements): - for repo_name, repo_provides in self.repo_provides.items(): - caps_classified[repo_name] = set() - for cap in requirements: - if " = " not in cap: - r = re.compile(str('^' + re.escape(cap) + '*')) + for reponame, base in self.repo_bases.items(): + q = base.sack.query() + pkg = q.filter(provides=[cap], arch=['noarch', ARCH], latest=True) + if pkg: + caps_classified.setdefault(reponame, set()) + caps_classified[reponame].add(cap) + break else: - r = re.compile(re.escape(cap)) - found = False - for repo_name, repo_provides in self.repo_provides.items(): - if list(filter(r.match, repo_provides)): - found = True - caps_classified[repo_name].add(cap) - if not found: caps_classified['result'].add(cap) @staticmethod @@ -192,7 +197,8 @@ class ModuleDepsDiffer(object): self._dump_pkgs_result(self.runtime_caps_classified.items()) def run(self): + self._init_repo_bases() self.get_package_requires() - self.get_repo_provides() + # self.get_repo_provides() self.classify_caps(self.build_caps_classified, self.build_req) self.classify_caps(self.runtime_caps_classified, self.runtime_req)