From 20bb96a17d99a09df762a09bfc4c07f0e8fe7ea1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mar 21 2018 23:49:03 +0000 Subject: [PATCH 1/3] Extend comps-sync more, add font packages `comps-sync.py` now has support for explicitly syncing *from* the workstation comps. In order to do this sanely though, we need a "blacklist" of things we don't want to sync. There are a few issues here: - desktop applications - dubious CLI apps - dnf - dubious misc things: e.g. `tcp_wrappers`, `crontabs` - arch-specific bits (not handled right now) (cherry picked from commit d0b52b1f195880997f7162d0d0a8c65aeda8b7ca) --- diff --git a/comps-sync-blacklist.yml b/comps-sync-blacklist.yml new file mode 100644 index 0000000..18746eb --- /dev/null +++ b/comps-sync-blacklist.yml @@ -0,0 +1,75 @@ +# This file has a list of packages to skip from comps that we don't want. + +# Entirely skip all packages in libreoffice +blacklist_groups: + - libreoffice + +blacklist: + core: + # We use rpm-ostree for the host + - dnf + - dnf-yum + - dnf-plugins-core + # Not sure why this is there at all + - dracut-config-rescue + # Eh...you can install this in a priv container + - parted + # https://bugzilla.redhat.com/show_bug.cgi?id=1452348 + - grubby + # Why? + - ncurses + # Needs to be arch specific + - ppc64-utils + base-x: + # These are for arm...need to handle arch-specific bits + - xorg-x11-drv-armsoc + - xorg-x11-drv-omap + gnome-desktop: + # We use rpm-ostree for the host + - PackageKit-command-not-found + - PackageKit-gtk3-module + # Apps + - evince + - evince-djvu + - evince-nautilus + - file-roller-nautilus + - gnome-calendar + - gnome-documents + - gnome-photos + workstation-product: + # We use rpm-ostree for the host + - dnf + - dnf-plugins-core + - deltarpm + # Really not worth listing explicitly + - filesystem + # Why? + - ncurses + - mailcap + # Really? + - tcp_wrappers + # This probably doesn't need to be default + - ppp + # We removed cronie a while ago, should nuke these too + - crontabs + - at + # Requires libreoffice + - unoconv + # Will change to git-core + - git + # Apps + - rhythmbox + # Random tools: container + - net-tools + - nmap-ncat + # filesystem tools, agian priv container + - dosfstools + - ntfs-3g + networkmanager-submodules: + # Let's use the builtin one by default + - dhcp-client + printing: + # We don't use PackageKit + - cups-pk-helper + # For now... + - ghostscript diff --git a/comps-sync.py b/comps-sync.py index 7c5e889..0b9fd9e 100755 --- a/comps-sync.py +++ b/comps-sync.py @@ -1,9 +1,11 @@ #!/usr/bin/python3 # Usage: ./comps-sync.py /path/to/comps-f28.xml.in -# Currently just *removes* packages from the manifest -# which are not mentioned in comps. +# +# Can both remove packages from the manifest +# which are not mentioned in comps, and add packages from +# comps. -import os, sys, subprocess, argparse, shlex, json +import os, sys, subprocess, argparse, shlex, json, yaml import libcomps def fatal(msg): @@ -20,6 +22,11 @@ base_pkgs_path = 'fedora-workstation-base-pkgs.json' with open(base_pkgs_path) as f: manifest = json.load(f) +with open('comps-sync-blacklist.yml') as f: + doc = yaml.load(f) + comps_blacklist = doc['blacklist'] + comps_blacklist_groups = doc['blacklist_groups'] + manifest_packages = set(manifest['packages']) comps_unknown = set() @@ -44,11 +51,16 @@ ws_environ = comps.environments['workstation-product-environment'] ws_pkgs = {} for gid in ws_environ.group_ids: group = comps.groups_match(id=gid.name)[0] + if gid.name in comps_blacklist_groups: + continue + blacklist = comps_blacklist.get(gid.name, set()) for pkg in group.packages: pkgname = pkg.name if pkg.type not in (libcomps.PACKAGE_TYPE_DEFAULT, libcomps.PACKAGE_TYPE_MANDATORY): continue + if pkgname in blacklist: + continue pkgdata = ws_pkgs.get(pkgname) if pkgdata is None: ws_pkgs[pkgname] = pkgdata = (pkg.type, set([gid.name])) @@ -72,6 +84,7 @@ ws_added = {} for (pkg,data) in ws_pkgs.items(): if pkg not in manifest_packages: ws_added[pkg] = data + manifest_packages.add(pkg) def format_pkgtype(n): if n == libcomps.PACKAGE_TYPE_DEFAULT: diff --git a/fedora-workstation-base-pkgs.json b/fedora-workstation-base-pkgs.json index 672d70f..1da2050 100644 --- a/fedora-workstation-base-pkgs.json +++ b/fedora-workstation-base-pkgs.json @@ -15,6 +15,7 @@ "NetworkManager-wwan", "PackageKit-gstreamer-plugin", "aajohan-comfortaa-fonts", + "abattis-cantarell-fonts", "adobe-source-han-sans-cn-fonts", "adobe-source-han-sans-tw-fonts", "adwaita-qt4", @@ -46,6 +47,7 @@ "cheese", "chrony", "cifs-utils", + "colord", "control-center", "coreutils", "cryptsetup", @@ -56,10 +58,13 @@ "dejavu-sans-mono-fonts", "dejavu-serif-fonts", "device-mapper-multipath", + "dhcp-client", "dmraid", + "dnsmasq", "dos2unix", "dracut-config-generic", "dracut-network", + "e2fsprogs", "ebtables", "efibootmgr", "eog", @@ -69,6 +74,7 @@ "fedora-release-workstation", "fedora-user-agent-chrome", "file-roller", + "filesystem", "firefox", "firewalld", "foomatic", @@ -81,6 +87,7 @@ "glib-networking", "glibc", "glibc-all-langpacks", + "glx-utils", "gnome-backgrounds", "gnome-bluetooth", "gnome-boxes", @@ -113,13 +120,26 @@ "gnu-free-sans-fonts", "gnu-free-serif-fonts", "gnupg2", + "google-noto-emoji-color-fonts", "google-noto-emoji-fonts", + "google-noto-sans-jp-fonts", + "google-noto-sans-kr-fonts", "google-noto-sans-lisu-fonts", "google-noto-sans-mandaic-fonts", "google-noto-sans-meetei-mayek-fonts", + "google-noto-sans-mono-cjk-jp-fonts", + "google-noto-sans-mono-cjk-kr-fonts", + "google-noto-sans-mono-cjk-sc-fonts", + "google-noto-sans-mono-cjk-tc-fonts", + "google-noto-sans-sc-fonts", "google-noto-sans-tagalog-fonts", "google-noto-sans-tai-tham-fonts", "google-noto-sans-tai-viet-fonts", + "google-noto-sans-tc-fonts", + "google-noto-serif-jp-fonts", + "google-noto-serif-kr-fonts", + "google-noto-serif-sc-fonts", + "google-noto-serif-tc-fonts", "gutenprint", "gutenprint-cups", "gvfs-afc", @@ -148,6 +168,7 @@ "ibus-typing-booster", "initscripts", "iproute", + "iptables", "iptstate", "iputils", "ipw2100-firmware", @@ -169,6 +190,7 @@ "iwl6050-firmware", "iwl7260-firmware", "jomolhari-fonts", + "julietaula-montserrat-fonts", "jwhois", "kbd", "kernel", @@ -177,10 +199,14 @@ "less", "libcanberra-gtk2", "libcanberra-gtk3", + "liberation-mono-fonts", + "liberation-sans-fonts", + "liberation-serif-fonts", "libertas-usb8388-firmware", "libproxy-mozjs", "libvirt-client", "libvirt-daemon-kvm", + "lklug-fonts", "logrotate", "lohit-assamese-fonts", "lohit-bengali-fonts", @@ -238,6 +264,7 @@ "rng-tools", "rootfiles", "rp-pppoe", + "rpm", "rsync", "rygel", "samba-client", From 03b305a1a98e280604f290b50b462194da75731d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mar 21 2018 23:50:03 +0000 Subject: [PATCH 2/3] comps-sync pass 3: Subset of workstation The mental model here is simpler if our base package set is truly a subset of Workstation. Extend `comps-sync` to also support deleting packages not in the Workstation set. The only exception here is that we still want `kernel-modules-extra` which for some reason is only in `livecd-tools`...I don't understand that. For now I added a `whitelist`. The set of resulting changes here generally makes sense. We lose some random unnecessary stuff like `teamd` and `dracut-network`, and `mactel-boot`. We gain things like the desktop background packages, `sshpass`, and some `qt` bits. ``` + NetworkManager-openconnect-gnome-1.2.4-9.fc28.x86_64 (fedora-rawhide) - NetworkManager-team-1:1.10.2-1.fc28.x86_64 (fedora-rawhide) + NetworkManager-ssh-1.2.7-1.fc28.x86_64 (fedora-rawhide) + NetworkManager-ssh-gnome-1.2.7-1.fc28.x86_64 (fedora-rawhide) - adobe-source-han-sans-cn-fonts-1.004-6.fc28.noarch (fedora-rawhide) - adobe-source-han-sans-tw-fonts-1.004-7.fc28.noarch (fedora-rawhide) + alsa-ucm-1.1.5-2.fc28.x86_64 (fedora-rawhide) - autogen-libopts-5.18.12-6.fc28.x86_64 (fedora-rawhide) + cyrus-sasl-plain-2.1.26-37.fc28.x86_64 (fedora-rawhide) + desktop-backgrounds-gnome-27.0.0-2.fc28.noarch (fedora-rawhide) - dmraid-1.0.0.rc16-37.fc28.x86_64 (fedora-rawhide) - dmraid-events-1.0.0.rc16-37.fc28.x86_64 (fedora-rawhide) - dracut-config-generic-046-92.git20180118.fc28.1.x86_64 (fedora-rawhide) - dracut-network-046-92.git20180118.fc28.1.x86_64 (fedora-rawhide) - fcoe-utils-1.0.32-3.fc27.x86_64 (fedora-rawhide) + f27-backgrounds-base-27.0.1-3.fc28.noarch (fedora-rawhide) + f27-backgrounds-gnome-27.0.1-3.fc28.noarch (fedora-rawhide) - fedora-productimg-workstation-27-2.fc28.x86_64 (fedora-rawhide) - gnutls-dane-3.6.2-1.fc28.x86_64 (fedora-rawhide) - gnutls-utils-3.6.2-1.fc28.x86_64 (fedora-rawhide) - hfsplus-tools-540.1.linux3-15.fc28.x86_64 (fedora-rawhide) - highlight-3.39-3.fc27.x86_64 (fedora-rawhide) - libconfig-1.5-9.fc28.x86_64 (fedora-rawhide) + libdnet-1.12-25.fc28.x86_64 (fedora-rawhide) + libmspack-0.6-0.2.alpha.fc28.x86_64 (fedora-rawhide) - libnl3-cli-3.4.0-3.fc28.x86_64 (fedora-rawhide) + libsane-hpaio-3.17.11-8.fc28.x86_64 (fedora-rawhide) - libteam-1.27-6.fc28.x86_64 (fedora-rawhide) - libvirt-client-4.0.0-1.fc28.x86_64 (fedora-rawhide) - lldpad-1.0.1-9.git036e314.fc28.x86_64 (fedora-rawhide) - mactel-boot-0.9-16.fc27.x86_64 (fedora-rawhide) - naver-nanum-fonts-common-3.020-20.20140930.fc28.noarch (fedora-rawhide) - naver-nanum-gothic-fonts-3.020-20.20140930.fc28.noarch (fedora-rawhide) + open-vm-tools-10.2.0-3.fc28.x86_64 (fedora-rawhide) + open-vm-tools-desktop-10.2.0-3.fc28.x86_64 (fedora-rawhide) + pinentry-gnome3-1.1.0-2.fc28.x86_64 (fedora-rawhide) + pulseaudio-module-x11-11.1-11.fc28.x86_64 (fedora-rawhide) + pulseaudio-utils-11.1-11.fc28.x86_64 (fedora-rawhide) + qt5-qtdeclarative-5.10.1-1.fc28.x86_64 (fedora-rawhide) + qt5-qtxmlpatterns-5.10.1-1.fc28.x86_64 (fedora-rawhide) + realmd-0.16.3-9.fc28.x86_64 (fedora-rawhide) - sgpio-1.2.0.10-20.fc28.x86_64 (fedora-rawhide) + sshpass-1.06-5.fc28.x86_64 (fedora-rawhide) - teamd-1.27-6.fc28.x86_64 (fedora-rawhide) - unbound-libs-1.6.8-3.fc28.x86_64 (fedora-rawhide) - vlgothic-fonts-20141206-10.fc28.noarch (fedora-rawhide) + virtualbox-guest-additions-5.2.6-4.fc28.x86_64 (fedora-rawhide) + xmlsec1-openssl-1.2.25-3.fc28.x86_64 (fedora-rawhide) ``` (cherry picked from commit 369c6d4e81a75aa87ee0706b22575c92a52d368d) --- diff --git a/comps-sync-blacklist.yml b/comps-sync-blacklist.yml index 18746eb..fe5af8f 100644 --- a/comps-sync-blacklist.yml +++ b/comps-sync-blacklist.yml @@ -1,4 +1,11 @@ -# This file has a list of packages to skip from comps that we don't want. +# This file has a list of packages to skip from comps that we don't want, plus +# a few whitelisted things. + +# For some reason today these are just in livecd-tools... +whitelist: + - kernel + - kernel-modules + - kernel-modules-extra # Entirely skip all packages in libreoffice blacklist_groups: @@ -53,12 +60,19 @@ blacklist: # We removed cronie a while ago, should nuke these too - crontabs - at + # This drags in a huge dependency set of dnf bits + - abrt-cli + - abrt-desktop + - abrt-java-connector # Requires libreoffice - unoconv # Will change to git-core - git # Apps - rhythmbox + - evolution + - evolution-ews + - evolution-help # Random tools: container - net-tools - nmap-ncat diff --git a/comps-sync.py b/comps-sync.py index 0b9fd9e..24532aa 100755 --- a/comps-sync.py +++ b/comps-sync.py @@ -25,6 +25,7 @@ with open(base_pkgs_path) as f: with open('comps-sync-blacklist.yml') as f: doc = yaml.load(f) comps_blacklist = doc['blacklist'] + comps_whitelist = doc['whitelist'] comps_blacklist_groups = doc['blacklist_groups'] manifest_packages = set(manifest['packages']) @@ -38,16 +39,12 @@ workstation_product_packages = set() # just the workstation environment. comps = libcomps.Comps() comps.fromxml_f(args.src) -for group in comps.groups: - for pkg in group.packages: - comps_packages.add(pkg.name) -for pkg in manifest_packages: - if pkg not in comps_packages: - comps_unknown.add(pkg) # Parse the workstation-product environment, gathering # default or mandatory packages. -ws_environ = comps.environments['workstation-product-environment'] +ws_env_name = 'workstation-product-environment' +ws_ostree_name = 'workstation-ostree-support' +ws_environ = comps.environments[ws_env_name] ws_pkgs = {} for gid in ws_environ.group_ids: group = comps.groups_match(id=gid.name)[0] @@ -69,12 +66,22 @@ for gid in ws_environ.group_ids: ws_pkgs[pkgname] = pkgdata = (pkg.type, pkgdata[1]) pkgdata[1].add(gid.name) +ws_ostree_pkgs = set() +for pkg in comps.groups_match(id=ws_ostree_name)[0].packages: + ws_ostree_pkgs.add(pkg.name) + +for pkg in manifest_packages: + if (pkg not in comps_whitelist and + pkg not in ws_pkgs and + pkg not in ws_ostree_pkgs): + comps_unknown.add(pkg) + # Look for packages in the manifest but not in comps at all n_manifest_new = len(comps_unknown) if n_manifest_new == 0: print("All manifest packages are already listed in comps.") else: - print("{} packages not in comps:".format(n_manifest_new)) + print("{} packages not in {}:".format(n_manifest_new, ws_env_name)) for pkg in sorted(comps_unknown): print(' ' + pkg) manifest_packages.remove(pkg) diff --git a/fedora-workstation-base-pkgs.json b/fedora-workstation-base-pkgs.json index 1da2050..6160a90 100644 --- a/fedora-workstation-base-pkgs.json +++ b/fedora-workstation-base-pkgs.json @@ -5,22 +5,21 @@ "NetworkManager", "NetworkManager-adsl", "NetworkManager-bluetooth", - "NetworkManager-config-connectivity-fedora", - "NetworkManager-openconnect", + "NetworkManager-openconnect-gnome", "NetworkManager-openvpn-gnome", "NetworkManager-pptp-gnome", - "NetworkManager-team", + "NetworkManager-ssh-gnome", "NetworkManager-vpnc-gnome", "NetworkManager-wifi", "NetworkManager-wwan", "PackageKit-gstreamer-plugin", "aajohan-comfortaa-fonts", "abattis-cantarell-fonts", - "adobe-source-han-sans-cn-fonts", - "adobe-source-han-sans-tw-fonts", + "acl", "adwaita-qt4", "adwaita-qt5", "alsa-plugins-pulseaudio", + "alsa-ucm", "alsa-utils", "at-spi2-atk", "at-spi2-core", @@ -50,29 +49,27 @@ "colord", "control-center", "coreutils", + "cpio", "cryptsetup", "cups", "cups-filters", "curl", + "cyrus-sasl-plain", + "dconf", "dejavu-sans-fonts", "dejavu-sans-mono-fonts", "dejavu-serif-fonts", - "device-mapper-multipath", + "desktop-backgrounds-gnome", "dhcp-client", - "dmraid", "dnsmasq", "dos2unix", - "dracut-config-generic", - "dracut-network", "e2fsprogs", - "ebtables", - "efibootmgr", "eog", "ethtool", - "fcoe-utils", "fedora-bookmarks", "fedora-release-workstation", "fedora-user-agent-chrome", + "file", "file-roller", "filesystem", "firefox", @@ -140,6 +137,9 @@ "google-noto-serif-kr-fonts", "google-noto-serif-sc-fonts", "google-noto-serif-tc-fonts", + "gstreamer1-plugin-mpg123", + "gstreamer1-plugins-bad-free", + "gstreamer1-plugins-good", "gutenprint", "gutenprint-cups", "gvfs-afc", @@ -150,11 +150,9 @@ "gvfs-gphoto2", "gvfs-mtp", "gvfs-smb", - "gzip", - "hfsplus-tools", - "highlight", "hostname", "hplip", + "hunspell", "hyperv-daemons", "ibus-gtk2", "ibus-gtk3", @@ -204,8 +202,8 @@ "liberation-serif-fonts", "libertas-usb8388-firmware", "libproxy-mozjs", - "libvirt-client", - "libvirt-daemon-kvm", + "librsvg2", + "libsane-hpaio", "lklug-fonts", "logrotate", "lohit-assamese-fonts", @@ -219,13 +217,10 @@ "lohit-telugu-fonts", "lrzsz", "lsof", - "lvm2", - "lzop", - "m17n-db", - "mactel-boot", "man-db", "man-pages", "mcelog", + "mdadm", "mesa-dri-drivers", "microcode_ctl", "mlocate", @@ -234,10 +229,11 @@ "mtr", "nautilus", "nautilus-sendto", - "naver-nanum-gothic-fonts", "nfs-utils", "nss-altfiles", "nss-mdns", + "ntfsprogs", + "open-vm-tools-desktop", "openssh-clients", "openssh-server", "orca", @@ -248,19 +244,29 @@ "passwd", "passwdqc", "pciutils", + "pinentry-gnome3", "pinfo", "plymouth", "plymouth-system-theme", "policycoreutils", "polkit", - "poppler-utils", "procps-ng", "psacct", + "pulseaudio", + "pulseaudio-module-x11", + "pulseaudio-utils", "qemu-guest-agent", - "qemu-kvm", "qgnomeplatform", - "radvd", + "qt", + "qt-settings", + "qt-x11", + "qt5-qtbase", + "qt5-qtbase-gui", + "qt5-qtdeclarative", + "qt5-qtxmlpatterns", + "quota", "rdist", + "realmd", "rng-tools", "rootfiles", "rp-pppoe", @@ -274,7 +280,6 @@ "setup", "setuptool", "shadow-utils", - "sheepdog", "sil-abyssinica-fonts", "sil-mingzat-fonts", "sil-nuosu-fonts", @@ -290,30 +295,34 @@ "symlinks", "system-config-printer-udev", "systemd", + "systemd-udev", "tabish-eeyek-fonts", "tar", "tcpdump", - "teamd", "telnet", "thai-scalable-waree-fonts", "time", "totem", "totem-nautilus", "traceroute", + "tracker", + "tracker-miners", "tree", + "unzip", "usb_modeswitch", "usbutils", "util-linux", "vconfig", "vim-minimal", - "vlgothic-fonts", + "virtualbox-guest-additions", "wget", "which", "wireless-tools", "words", "wvdial", + "xdg-desktop-portal", + "xdg-desktop-portal-gtk", "xdg-user-dirs-gtk", - "xfsprogs", "xorg-x11-drv-ati", "xorg-x11-drv-evdev", "xorg-x11-drv-fbdev", @@ -329,7 +338,6 @@ "xorg-x11-utils", "xorg-x11-xauth", "xorg-x11-xinit", - "xz", "yelp", "zd1211-firmware", "zip" From cae0ace60db95355250e0c3a86f13f7c734ecd94 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Mar 21 2018 23:51:31 +0000 Subject: [PATCH 3/3] comps sync 3 packages not in workstation-product-environment: caribou caribou-gtk2-module caribou-gtk3-module 1 packages not in manifest: simple-scan (mandatory, groups: gnome-desktop) Wrote fedora-workstation-base-pkgs.json (cherry picked from commit b73c11675ded341044b55dcef8adc87b48676851) --- diff --git a/fedora-workstation-base-pkgs.json b/fedora-workstation-base-pkgs.json index 6160a90..be6a44b 100644 --- a/fedora-workstation-base-pkgs.json +++ b/fedora-workstation-base-pkgs.json @@ -40,9 +40,6 @@ "bridge-utils", "btrfs-progs", "bzip2", - "caribou", - "caribou-gtk2-module", - "caribou-gtk3-module", "cheese", "chrony", "cifs-utils", @@ -284,6 +281,7 @@ "sil-mingzat-fonts", "sil-nuosu-fonts", "sil-padauk-fonts", + "simple-scan", "smc-meera-fonts", "sos", "spice-vdagent",