From fa798595cf05afac440ae54a0922213f89fcf74e Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Oct 31 2022 23:35:40 +0000 Subject: [PATCH 1/2] comps-sync.py: handle arch-dependent package specs (#311) comps allows for a package to be included in a group only on specified arches, with a line like this: kio-gdrive up till now, `comps-sync.py` effectively ignored this, treating such a line exactly the same as this: kio-gdrive and attempting to include the package even when building on e.g. ppc64le. Solving this is unfortunately tricky due to exactly how libcomps handles these arch restrictions. It does not expose the list of arches as a property of each returned 'package' object when you do a group search. Instead you're supposed to filter the group search down to the arch(es) you are interested in, and it only includes the appropriate packages for that arch. So all we can do is run our queries multiple times, once for each arch we may wish to build on, keep track of the results per arch as we go, and do some fancy footwork to keep track of the mapping between "archful" comps entries and the lists in the YAML files. Signed-off-by: Adam Williamson --- diff --git a/comps-sync.py b/comps-sync.py index 54d3040..4b9bfdb 100755 --- a/comps-sync.py +++ b/comps-sync.py @@ -8,6 +8,8 @@ import os, sys, subprocess, argparse, shlex, json, yaml, re import libcomps +ARCHES = ("x86_64", "aarch64", "ppc64le") + def fatal(msg): print >>sys.stderr, msg sys.exit(1) @@ -26,8 +28,13 @@ def write_manifest(fpath, pkgs, include=None): if include is not None: f.write("include: {}\n".format(include)) f.write("packages:\n") - for pkg in sorted(pkgs): + for pkg in sorted(pkgs['all']): f.write(" - {}\n".format(pkg)) + for arch in ARCHES: + if pkgs[arch]: + f.write(f"packages-{arch}:\n") + for pkg in sorted(pkgs[arch]): + f.write(" - {}\n".format(pkg)) print("Wrote {}".format(fpath)) parser = argparse.ArgumentParser() @@ -41,7 +48,13 @@ print("Syncing packages common to all desktops:") base_pkgs_path = 'fedora-common-ostree-pkgs.yaml' with open(base_pkgs_path) as f: manifest = yaml.safe_load(f) -manifest_packages = set(manifest['packages']) +manifest_packages = {} +manifest_packages['all'] = set(manifest['packages']) +for arch in ARCHES: + if f'packages-{arch}' in manifest: + manifest_packages[arch] = set(manifest[f'packages-{arch}']) + else: + manifest_packages[arch] = set() with open('comps-sync-exclude-list.yml') as f: doc = yaml.safe_load(f) @@ -70,24 +83,27 @@ ws_ostree_name = 'workstation-ostree-support' ws_environ = comps.environments[ws_env_name] ws_pkgs = {} for gid in ws_environ.group_ids: - group = comps.groups_match(id=gid.name)[0] if gid.name in comps_exclude_list_groups: continue exclude_list = comps_exclude_list.get(gid.name, set()) - for pkg in group.packages: - pkgname = pkg.name - if pkg.type not in (libcomps.PACKAGE_TYPE_DEFAULT, - libcomps.PACKAGE_TYPE_MANDATORY): - continue - if pkgname in exclude_list or is_exclude_listed(pkgname): - continue - pkgdata = ws_pkgs.get(pkgname) - if pkgdata is None: - ws_pkgs[pkgname] = pkgdata = (pkg.type, set([gid.name])) - if (pkgdata[0] == libcomps.PACKAGE_TYPE_DEFAULT and - pkg.type == libcomps.PACKAGE_TYPE_MANDATORY): - ws_pkgs[pkgname] = pkgdata = (pkg.type, pkgdata[1]) - pkgdata[1].add(gid.name) + for arch in ARCHES: + filtered = comps.arch_filter([arch]) + group = filtered.groups_match(id=gid.name)[0] + for pkg in group.packages: + pkgname = pkg.name + if pkg.type not in (libcomps.PACKAGE_TYPE_DEFAULT, + libcomps.PACKAGE_TYPE_MANDATORY): + continue + if pkgname in exclude_list or is_exclude_listed(pkgname): + continue + pkgdata = ws_pkgs.get(pkgname) + if pkgdata is None: + ws_pkgs[pkgname] = pkgdata = (pkg.type, set([gid.name]), set([arch])) + if (pkgdata[0] == libcomps.PACKAGE_TYPE_DEFAULT and + pkg.type == libcomps.PACKAGE_TYPE_MANDATORY): + ws_pkgs[pkgname] = pkgdata = (pkg.type, pkgdata[1], pkgdata[2]) + pkgdata[1].add(gid.name) + pkgdata[2].add(arch) ws_ostree_pkgs = set() for pkg in comps.groups_match(id=ws_ostree_name)[0].packages: @@ -95,11 +111,17 @@ for pkg in comps.groups_match(id=ws_ostree_name)[0].packages: ws_ostree_pkgs.add(pkg.name) comps_unknown = set() -for pkg in manifest_packages: - if (pkg not in comps_include_list and - pkg not in ws_pkgs and - pkg not in ws_ostree_pkgs): - comps_unknown.add(pkg) +for arch in manifest_packages: + for pkg in manifest_packages[arch]: + if arch == "all": + if pkg in ws_pkgs and set(ws_pkgs[pkg][2]) == set(ARCHES): + continue + else: + if pkg in ws_pkgs and arch in ws_pkgs[pkg][2]: + continue + if (pkg not in comps_include_list and + pkg not in ws_ostree_pkgs): + comps_unknown.add((pkg, arch)) # Look for packages in the manifest but not in comps at all n_manifest_new = len(comps_unknown) @@ -107,16 +129,25 @@ if n_manifest_new == 0: print(" - All manifest packages are already listed in comps.") else: print(" - {} packages not in {}:".format(n_manifest_new, ws_env_name)) - for pkg in sorted(comps_unknown): - print(' {}'.format(pkg)) - manifest_packages.remove(pkg) + for (pkg, arch) in sorted(comps_unknown, key = lambda x: x[0]): + print(' {} (arch: {})'.format(pkg, arch)) + manifest_packages[arch].remove(pkg) # Look for packages in workstation but not in the manifest ws_added = {} for (pkg,data) in ws_pkgs.items(): - if pkg not in manifest_packages: - ws_added[pkg] = data - manifest_packages.add(pkg) + if set(ARCHES) == set(data[2]): + if pkg not in manifest_packages['all']: + ws_added[pkg] = data + manifest_packages['all'].add(pkg) + else: + for arch in data[2]: + if pkg not in manifest_packages[arch]: + manifest_packages[arch].add(pkg) + if pkg not in ws_added: + ws_added[pkg] = (data[0], data[1], set([arch])) + else: + ws_added[pkg][2].add(arch) n_comps_new = len(ws_added) if n_comps_new == 0: @@ -124,8 +155,8 @@ if n_comps_new == 0: else: print(" - {} packages not in manifest:".format(n_comps_new)) for pkg in sorted(ws_added): - (req, groups) = ws_added[pkg] - print(' {} ({}, groups: {})'.format(pkg, format_pkgtype(req), ', '.join(groups))) + (req, groups, arches) = ws_added[pkg] + print(' {} ({}, groups: {}, arches: {})'.format(pkg, format_pkgtype(req), ', '.join(groups), ', '.join(arches))) if (n_manifest_new > 0 or n_comps_new > 0) and args.save: write_manifest(base_pkgs_path, manifest_packages) @@ -139,37 +170,65 @@ for desktop in [ 'gnome-desktop', 'kde-desktop', 'xfce-desktop', manifest_path = '{}-pkgs.yaml'.format(desktop) with open(manifest_path) as f: manifest = yaml.safe_load(f) - manifest_packages = set(manifest['packages']) + manifest_packages = {} + manifest_packages['all'] = set(manifest['packages']) + for arch in ARCHES: + if f'packages-{arch}' in manifest: + manifest_packages[arch] = set(manifest[f'packages-{arch}']) + else: + manifest_packages[arch] = set() # Filter packages in the comps desktop group using the exclude_list - comps_group_pkgs = set() - for pkg in comps.groups_match(id=desktop)[0].packages: - pkgname = pkg.name - exclude_list = comps_desktop_exclude_list.get(desktop, set()) - if pkgname in exclude_list or is_exclude_listed(pkgname): - continue - comps_group_pkgs.add(pkg.name) - - # Look for packages in the manifest but not in comps group - comps_unknown = set() - for pkg in manifest_packages: - if pkg not in comps_group_pkgs: - comps_unknown.add(pkg) + comps_group_pkgs = {} + for arch in ARCHES: + filtered = comps.arch_filter([arch]) + for pkg in filtered.groups_match(id=desktop)[0].packages: + pkgname = pkg.name + exclude_list = comps_desktop_exclude_list.get(desktop, set()) + if pkgname in exclude_list or is_exclude_listed(pkgname): + continue + if pkgname in comps_group_pkgs: + comps_group_pkgs[pkgname].add(arch) + else: + comps_group_pkgs[pkgname] = set([arch]) + comps_unknown = set() + for arch in manifest_packages: + for pkg in manifest_packages[arch]: + if arch == "all": + if pkg in comps_group_pkgs and set(comps_group_pkgs[pkg]) == set(ARCHES): + continue + else: + if pkg in comps_group_pkgs and arch in comps_group_pkgs[pkg]: + continue + comps_unknown.add((pkg, arch)) + + # Look for packages in the manifest but not in comps at all n_manifest_new = len(comps_unknown) if n_manifest_new == 0: print(" - All manifest packages are already listed in comps.") else: - print(" - {} packages not in {} comps group:".format(n_manifest_new, desktop)) - for pkg in sorted(comps_unknown): - print(' {}'.format(pkg)) - manifest_packages.remove(pkg) + print(" - {} packages not in {}:".format(n_manifest_new, ws_env_name)) + for (pkg, arch) in sorted(comps_unknown, key = lambda x: x[0]): + print(' {} (arch: {})'.format(pkg, arch)) + manifest_packages[arch].remove(pkg) + # Look for packages in comps but not in the manifest - desktop_pkgs_added = set() - for pkg in comps_group_pkgs: - if pkg not in manifest_packages: - desktop_pkgs_added.add(pkg) + desktop_pkgs_added = {} + for (pkg, parches) in comps_group_pkgs.items(): + if set(ARCHES) == set(parches): + if pkg not in manifest_packages['all']: + desktop_pkgs_added[pkg] = parches + manifest_packages['all'].add(pkg) + else: + for arch in parches: + if pkg not in manifest_packages[arch]: + manifest_packages[arch].add(pkg) + if pkg not in desktop_pkgs_added: + desktop_pkgs_added[pkg] = set([arch]) + else: + desktop_pkgs_added[pkg].add(arch) n_comps_new = len(desktop_pkgs_added) if n_comps_new == 0: @@ -177,8 +236,8 @@ for desktop in [ 'gnome-desktop', 'kde-desktop', 'xfce-desktop', else: print(" - {} packages not in {} manifest:".format(n_comps_new, desktop)) for pkg in sorted(desktop_pkgs_added): - print(' {}'.format(pkg)) - manifest_packages.add(pkg) + arches = desktop_pkgs_added[pkg] + print(' {} (arches: {})'.format(pkg, ', '.join(arches))) # Update manifest if (n_manifest_new > 0 or n_comps_new > 0) and args.save: From edb7d86d390cd10c5f33774cbfa24c1c68e0aecf Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Oct 31 2022 23:35:44 +0000 Subject: [PATCH 2/2] comps-sync f38 2022-10-31 This includes corrections for per-arch packages, thanks to the improvements to comps-sync.py in the previous commit. Signed-off-by: Adam Williamson --- diff --git a/deepin-desktop-pkgs.yaml b/deepin-desktop-pkgs.yaml index 8d264cb..82e8f1e 100644 --- a/deepin-desktop-pkgs.yaml +++ b/deepin-desktop-pkgs.yaml @@ -1,7 +1,6 @@ # DO NOT EDIT! This content is generated from comps-sync.py include: fedora-common-ostree.yaml packages: - - chromium - deepin-calculator - deepin-calendar - deepin-desktop @@ -13,3 +12,7 @@ packages: - deepin-screenshot - deepin-system-monitor - firewall-config +packages-x86_64: + - chromium +packages-aarch64: + - chromium diff --git a/fedora-common-ostree-pkgs.yaml b/fedora-common-ostree-pkgs.yaml index e243771..1ef3fbf 100644 --- a/fedora-common-ostree-pkgs.yaml +++ b/fedora-common-ostree-pkgs.yaml @@ -8,7 +8,6 @@ packages: - abattis-cantarell-vf-fonts - acl - adwaita-qt5 - - alsa-sof-firmware - alsa-ucm - alsa-utils - amd-gpu-firmware @@ -236,3 +235,5 @@ packages: - zd1211-firmware - zip - zram-generator-defaults +packages-x86_64: + - alsa-sof-firmware diff --git a/kde-desktop-pkgs.yaml b/kde-desktop-pkgs.yaml index 78ef2a9..f2118b3 100644 --- a/kde-desktop-pkgs.yaml +++ b/kde-desktop-pkgs.yaml @@ -23,7 +23,6 @@ packages: - khelpcenter - khotkeys - kinfocenter - - kio-gdrive - kmag - kmenuedit - kmousetool @@ -65,3 +64,7 @@ packages: - systemd-oomd-defaults - xdg-desktop-portal-gnome - xorg-x11-drv-libinput +packages-x86_64: + - kio-gdrive +packages-aarch64: + - kio-gdrive diff --git a/lxqt-desktop-pkgs.yaml b/lxqt-desktop-pkgs.yaml index 2275ab2..b0b60c9 100644 --- a/lxqt-desktop-pkgs.yaml +++ b/lxqt-desktop-pkgs.yaml @@ -4,7 +4,6 @@ packages: - breeze-cursor-theme - breeze-gtk - breeze-icon-theme - - falkon - firewall-config - initial-setup-gui - lxqt-about @@ -33,3 +32,7 @@ packages: - sddm-themes - upower - xdg-user-dirs +packages-x86_64: + - falkon +packages-aarch64: + - falkon