From abd56aaf71c0c278b8acb31c7b7ddf69ad3dfd16 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Jun 02 2023 09:52:22 +0000 Subject: [PATCH 1/5] Add support for ostree desktop variants Example failure message: ``` [FAIL] Ostree installer (variant Onyx, arch x86_64) failed, but going on anyway. ``` Needs: https://pagure.io/fedora-kickstarts/pull-request/961 --- diff --git a/compose_tracker.py b/compose_tracker.py index 1b83b38..48c1bea 100755 --- a/compose_tracker.py +++ b/compose_tracker.py @@ -199,11 +199,14 @@ class Consumer(object): maintainers_info = toml.loads(req.text) try: if line != 'eln': - matching_lists = re.findall(r'variant (\w+), arch (\S+), subvariant (\w+)', line) + matching_lists = re.findall(r'variant (\w+), arch (\S+)(, subvariant (\w+))*', line) if matching_lists: for ml in matching_lists: # For all arches if ml[1] == '*': + # Special case for ostree variants that do not have a subvariant + if not ml[2]: + ml[2] = ml[0] # Get the list of all maintainers for all arches for that variant and sub variant maintainers_list = list(maintainers_info.get(ml[0].lower()).get(ml[2].lower()).values()) # Get the list of all fas usernames from the above list From 0122822c86072882e6352bd6f10ec7e44e9266ce Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Nov 10 2023 17:58:18 +0000 Subject: [PATCH 2/5] Fix ostree/ostree installer variants support Also fixes the logic for Spins broken in the previous change. See: https://pagure.io/fedora-kickstarts/pull-request/961 Fixes: abd56aa Add support for ostree desktop variants --- diff --git a/compose_tracker.py b/compose_tracker.py index 48c1bea..602f735 100755 --- a/compose_tracker.py +++ b/compose_tracker.py @@ -199,29 +199,30 @@ class Consumer(object): maintainers_info = toml.loads(req.text) try: if line != 'eln': - matching_lists = re.findall(r'variant (\w+), arch (\S+)(, subvariant (\w+))*', line) + matching_lists = re.findall(r'variant (\w+), arch ([0-9a-zA-Z_\*]+)(, subvariant (\w+))?', line) if matching_lists: + # Special case for ostree variants that do not have a subvariant for ml in matching_lists: - # For all arches - if ml[1] == '*': - # Special case for ostree variants that do not have a subvariant - if not ml[2]: - ml[2] = ml[0] - # Get the list of all maintainers for all arches for that variant and sub variant - maintainers_list = list(maintainers_info.get(ml[0].lower()).get(ml[2].lower()).values()) + variant = ml[0] + arches = ml[1] + subvariant = ml[3] + logger.info("Found variant %s, arches: %s, subvariant: %s", variant, arches, subvariant) + maintainers_tpm = maintainers_info.get(variant.lower()) + if subvariant != '': + maintainers_tpm = maintainers_tpm.get(subvariant.lower()) + if arches == '*': + # Get the list of all maintainers for all arches + maintainers_list = list(maintainers_tpm.values()) # Get the list of all fas usernames from the above list maintainer_fas_list = [maintainer.get('fas') for maintainer in maintainers_list] # Since there can be repetitions of fas usernames, create a set of usernames maintainers = set(itertools.chain(*maintainer_fas_list)) - # return the list of maintainers, variant and sub variant - return maintainers, ml[0], ml[2] else: - # Get the list of all maintainers for the variant, sub variant and arch - maintainers_list = maintainers_info.get(ml[0].lower()).get(ml[2].lower()).get(ml[1].lower()) + # Get the list of all maintainers for a specifc arch + maintainers_list = maintainers_tpm.get(arches.lower()) # Get a set of fas usernames from the above list maintainers = set(maintainers_list.get('fas')) - # return the list of maintainers, variant and sub variant - return maintainers, ml[0], ml[2] + return maintainers, variant, subvariant else: # Get the list of all maintainers for ELN maintainers_list = maintainers_info.get(line.lower()) From 81360b37e9c4bdf2883acf896d2eec06b928bed2 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Nov 10 2023 17:58:49 +0000 Subject: [PATCH 3/5] tests: Add ostree & ostree installer test cases --- diff --git a/test_consumer.py b/test_consumer.py index 7375848..f5651e5 100644 --- a/test_consumer.py +++ b/test_consumer.py @@ -52,6 +52,18 @@ EXAMPLE_PUNGI_LOG_PING_MAINTAINERS = """2020-02-04 05:59:24 [INFO ] [BEGIN] - 2020-02-04 06:03:40 [LIVE_MEDIA ] [ERROR ] [FAIL] Live media (variant Spins, arch x86_64, subvariant xfce) failed, but going on anyway. 2020-02-04 06:03:40 [LIVE_MEDIA ] [ERROR ] [FAIL] Live media (variant Spins, arch x86_64, subvariant lxqt) failed, but going on anyway. 2020-02-04 06:04:44 [INFO ] [DONE ] ---------- PHASE: LIVE MEDIA ---------- +2020-02-04 05:59:24 [INFO ] [BEGIN] ---------- PHASE: OSTREE ---------- +2020-02-04 06:03:08 [ERROR ] [FAIL] Ostree (variant Silverblue, arch x86_64) failed, but going on anyway. +2020-02-04 06:03:08 [ERROR ] Runroot task failed: 413508211. See /mnt/koji/compose/rawhide/Fedora-Rawhide-20200204.n.0/logs/x86_64/Silverblue/ostree-1/runroot.log for more details. +2020-02-04 06:03:40 [ERROR ] [FAIL] Ostree (variant Silverblue, arch ppc64le) failed, but going on anyway. +2020-02-04 06:03:40 [ERROR ] Runroot task failed: 413508221. See /mnt/koji/compose/rawhide/Fedora-Rawhide-20200204.n.0/logs/ppc64le/Silverblue/ostree-2/runroot.log for more details. +2020-02-04 06:04:44 [INFO ] [DONE ] ---------- PHASE: OSTREE ---------- +2023-11-10 06:46:59 [INFO ] [BEGIN] ---------- PHASE: OSTREE_INSTALLER ---------- +2023-11-10 07:23:08 [OSTREE_INSTALLER] [ERROR ] [FAIL] Ostree installer (variant Kinoite, arch ppc64le) failed, but going on anyway. +2023-11-10 07:23:08 [OSTREE_INSTALLER] [ERROR ] Runroot task failed: 108839426. See /mnt/koji/compose/rawhide/Fedora-Rawhide-20231110.n.0/logs/ppc64le/Kinoite/ostree_installer-2/runroot.log for more details. +2023-11-10 07:18:12 [OSTREE_INSTALLER] [ERROR ] [FAIL] Ostree installer (variant Kinoite, arch aarch64) failed, but going on anyway. +2023-11-10 07:18:12 [OSTREE_INSTALLER] [ERROR ] Runroot task failed: 108839504. See /mnt/koji/compose/rawhide/Fedora-Rawhide-20231110.n.0/logs/aarch64/Kinoite/ostree_installer-1/runroot.log for more details. +2023-11-10 07:50:26 [INFO ] [DONE ] ---------- PHASE: OSTREE_INSTALLER ---------- """ EXAMPLE_KS_MAINTAINERS = """ @@ -62,6 +74,26 @@ maintainers = [ "Peter Robinson" ] [spins.xfce.x86_64] fas = [ "nonamedotc", "kevin", "maxamillion" ] maintainers = [ "Mukundan Ragavan", "Kevin Fenzi", "Adam Miller" ] + +[silverblue.x86_64] +fas = [ "siosm" ] +maintainers = [ "Timothée Ravier" ] +[silverblue.aarch64] +fas = [ "siosm" ] +maintainers = [ "Timothée Ravier" ] +[silverblue.ppc64le] +fas = [ "siosm" ] +maintainers = [ "Timothée Ravier" ] + +[kinoite.x86_64] +fas = [ "siosm" ] +maintainers = [ "Timothée Ravier" ] +[kinoite.aarch64] +fas = [ "siosm" ] +maintainers = [ "Timothée Ravier" ] +[kinoite.ppc64le] +fas = [ "siosm" ] +maintainers = [ "Timothée Ravier" ] """ @@ -250,6 +282,9 @@ def test_consumer_maintainer_pings(mocker, caplog): assert "Variant: Spins, subvariant: SoaS task failed. Pinging maintainers: @pbrobinson" in caplog.text # Cannot assert particular maintainers since the set order is random assert "Variant: Spins, subvariant: xfce task failed. Pinging maintainers" in caplog.text and "@kevin" in caplog.text and '@nonamedotc' in caplog.text and '@maxamillion' in caplog.text + # ostree & ostree installer cases + assert "Variant: Silverblue, subvariant: task failed. Pinging maintainers: @siosm" in caplog.text + assert "Variant: Kinoite, subvariant: task failed. Pinging maintainers: @siosm" in caplog.text def test_log_urls(mocker, caplog): "Test that we get the correct labels from the fedora-messaging message." From 42518169e9b96982dd2036bb46b032a02ca5e95b Mon Sep 17 00:00:00 2001 From: James Antill Date: Nov 14 2024 20:45:15 +0000 Subject: [PATCH 4/5] Move ping maintainers code out of the log parsing loop. --- diff --git a/compose_tracker.py b/compose_tracker.py index 602f735..ca74332 100755 --- a/compose_tracker.py +++ b/compose_tracker.py @@ -328,6 +328,25 @@ class Consumer(object): lines = [] pass + sent_ping = False + def _ping_maintainers(line): + sent_ping = True + if not eln: + maintainers, variant, sub_variant = self.get_maintainers(line, ks_url) or (None, None, None) + else: + maintainers, line = self.get_maintainers('eln', ks_url) or (None, None, None) + if not maintainers: + logger.info("No maintainers info available. The ticket will be filed without pinging any maintainers") + else: + if not eln: + ping_line = f'Variant: {variant}, subvariant: {sub_variant} task failed. Pinging maintainers: ' + else: + ping_line = f'ELN compose has failed. Pinging maintainers: ' + for maintainer in maintainers: + ping_line += f'@{maintainer} ' + content += ping_line + '\n\n' + logger.info(ping_line) + phases = {} for x in range(1, len(lines)): linedate = lines[x-1][:19] # keep only the date for each line @@ -369,21 +388,7 @@ class Consumer(object): if self.config["ks_repo"]: # Ping the maintainers - if not eln: - maintainers, variant, sub_variant = self.get_maintainers(line, ks_url) or (None, None, None) - else: - maintainers, line = self.get_maintainers('eln', ks_url) or (None, None, None) - if not maintainers: - logger.info("No maintainers info available. The ticket will be filed without pinging any maintainers") - else: - if not eln: - ping_line = f'Variant: {variant}, subvariant: {sub_variant} task failed. Pinging maintainers: ' - else: - ping_line = f'ELN compose has failed. Pinging maintainers: ' - for maintainer in maintainers: - ping_line += f'@{maintainer} ' - content += ping_line + '\n\n' - logger.info(ping_line) + _ping_maintainers(line) # If this is the Compose run failed line, then add it # to the description too From 3301404157f52688f2a27511cdc1d2eb84c546ce Mon Sep 17 00:00:00 2001 From: James Antill Date: Nov 14 2024 20:52:27 +0000 Subject: [PATCH 5/5] Make sure ELN gets pings about entire failures. --- diff --git a/compose_tracker.py b/compose_tracker.py index ca74332..61372cd 100755 --- a/compose_tracker.py +++ b/compose_tracker.py @@ -397,6 +397,10 @@ class Consumer(object): content+= f'- Compose run failed because: {text}\n' content+= "```\n%s\n```\n" % (line) + if eln and not sent_ping: # ELN fails as a single unit, so make sure pings happen + ks_url = self.config["ks_repo"] + '/raw/main/f/maintainers.toml' + _ping_maintainers('') + compose_start = dt.datetime.fromisoformat(lines[0][:19]) compose_end = dt.datetime.fromisoformat(lines[-1][:19]) content+= f"Compose Total time: {compose_end - compose_start}\n"