From 9bac538b750194280384fdd8fe79f0fb865fdfb3 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Nov 11 2021 19:40:29 +0000 Subject: Adding compose tracking for ELN composes Signed-off-by: Mohan Boddu --- diff --git a/compose-tracker.toml b/compose-tracker.toml index db0713d..b59ae1b 100644 --- a/compose-tracker.toml +++ b/compose-tracker.toml @@ -36,7 +36,7 @@ arguments = {} [[bindings]] queue = "00000000-0000-0000-0000-000000000000" exchange = "amq.topic" -routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change"] +routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change", "org.fedoraproject.prod.odcs.compose.state-changed"] [consumer_config] composes_to_skip = ["IoT"] diff --git a/compose_tracker.py b/compose_tracker.py index 79bde22..73188cc 100755 --- a/compose_tracker.py +++ b/compose_tracker.py @@ -148,37 +148,46 @@ class Consumer(object): return None maintainers_info = toml.loads(req.text) try: - matching_lists = re.findall(r'variant (\w+), arch (\S+), subvariant (\w+)', line) - if matching_lists: - for ml in matching_lists: - # For all arches - if ml[1] == '*': - # Get the list of all maintainers for all arches for that variant and sub variant - maintainers_list = list(maintainers_info.get(ml[0].lower()).get(ml[2].lower()).values()) - # Get the list of all fas usernames from the above list - maintainer_fas_list = [maintainer.get('fas') for maintainer in maintainers_list] - # Since there can be repetitions of fas usernames, create a set of usernames - maintainers = set(itertools.chain(*maintainer_fas_list)) - # return the list of maintainers, variant and sub variant - return maintainers, ml[0], ml[2] - else: - # Get the list of all maintainers for the variant, sub variant and arch - maintainers_list = maintainers_info.get(ml[0].lower()).get(ml[2].lower()).get(ml[1].lower()) - # Get a set of fas usernames from the above list - maintainers = set(maintainers_list.get('fas')) - # return the list of maintainers, variant and sub variant - return maintainers, ml[0], ml[2] + if line != 'eln': + matching_lists = re.findall(r'variant (\w+), arch (\S+), subvariant (\w+)', line) + if matching_lists: + for ml in matching_lists: + # For all arches + if ml[1] == '*': + # Get the list of all maintainers for all arches for that variant and sub variant + maintainers_list = list(maintainers_info.get(ml[0].lower()).get(ml[2].lower()).values()) + # Get the list of all fas usernames from the above list + maintainer_fas_list = [maintainer.get('fas') for maintainer in maintainers_list] + # Since there can be repetitions of fas usernames, create a set of usernames + maintainers = set(itertools.chain(*maintainer_fas_list)) + # return the list of maintainers, variant and sub variant + return maintainers, ml[0], ml[2] + else: + # Get the list of all maintainers for the variant, sub variant and arch + maintainers_list = maintainers_info.get(ml[0].lower()).get(ml[2].lower()).get(ml[1].lower()) + # Get a set of fas usernames from the above list + maintainers = set(maintainers_list.get('fas')) + # return the list of maintainers, variant and sub variant + return maintainers, ml[0], ml[2] + else: + # Get the list of all maintainers for ELN + maintainers_list = maintainers_info.get(line.lower()) + # Get a set of fas usernames from the above list + maintainers = set(maintainers_list.get('fas')) + # return the list of interested parties for eln + return maintainers, line + except Exception: logger.info("No maintainer info found, skipping") return None - def __call__(self, message: fedora_messaging.api.Message): + def __call__(self, message: fedora_messaging.api.Message, eln=False): # Catch any exceptions and don't raise them further because # it will cause /usr/bin/fedora-messaging to crash and we'll # lose the traceback logs from the container try: - self.process(message) + self.process(message, eln) except Exception as e: logger.error('Caught Exception!') logger.error('###################################') @@ -187,13 +196,17 @@ class Consumer(object): logger.error('\t continuing...') pass - def process(self, message: fedora_messaging.api.Message): + def process(self, message: fedora_messaging.api.Message, eln): logger.debug(message.topic) logger.debug(message.body) # Grab the raw message body and the status from that - msg = message.body - status = msg['status'] + if not eln: + msg = message.body + status = msg['status'] + else: + msg = message.body['compose'] + status = msg['state_name'] # Print out a log statement if the day has changed today = dt.date.today() @@ -203,28 +216,44 @@ class Consumer(object): logger.info('mark') # If we are in good states then continue - if status in ['FINISHED', 'STARTED']: - # Print out an indicator of progress (i.e. composes have been - # finishing and are successful. Note this won't print out - # right unless we are setting PYTHONUNBUFFERED env var - # to a non-empty string - print('.', end='') # some sort of indicator of progress - return + if not eln: + if status in ['FINISHED', 'STARTED']: + # Print out an indicator of progress (i.e. composes have been + # finishing and are successful. Note this won't print out + # right unless we are setting PYTHONUNBUFFERED env var + # to a non-empty string + print('.', end='') # some sort of indicator of progress + return + else: + if status not in ['failed']: + return print('') # get to the next line - # We have a compose that either failed or had missing artifacts - # create a new issue. - title = msg['compose_id'] + ' ' + status - logfileurl = msg['location'] + '/../logs/global/pungi.global.log' - logger.info("%s\t%s" % (title, logfileurl)) + if not eln: + # We have a compose that either failed or had missing artifacts + # create a new issue. + title = msg['compose_id'] + ' ' + status + logfileurl = msg['location'] + '/../logs/global/pungi.global.log' + logger.info("%s\t%s" % (title, logfileurl)) + else: + title = msg['pungi_compose_id'] + ' ' + status + logfileurl = msg['toplevel_url'] + '/logs/global/pungi.global.log' + logger.info("%s\t%s" % (title, logfileurl)) + # It was requested that we not file issues for IoT composes # https://pagure.io/releng/failed-composes/issue/39#comment-591054 for compose in self.config["composes_to_skip"]: - if compose in msg["compose_id"]: - logger.info(f"Skipping filing issues for {compose} composes") - return + if not eln: + if compose in msg["compose_id"]: + logger.info(f"Skipping filing issues for {compose} composes") + return + else: + if compose in msg["pungi_compose_id"]: + logger.info(f"Skipping filing issues for {compose} composes") + return + # variable to hold description for issue content = "[pungi.global.log](%s)\n\n" % logfileurl @@ -260,15 +289,18 @@ class Consumer(object): # next line and add them in markdown format. Also grab # the taskid if we can and print a hyperlink to koji if re.search(r'\[FAIL\]', line): - # Find the maintainers in kickstarts repo under - # mainatiners.toml file and ping them in the ticket - # Find the branch in kickstarts repo - if self.config["ks_repo"]: - if 'rawhide' == msg['release_version'].lower(): - ks_branch = 'main' - elif 'epel' not in msg['release_name'].lower(): - ks_branch = 'f' + msg['release_version'] - ks_url = self.config["ks_repo"] + '/raw/' + ks_branch + '/f/maintainers.toml' + if not eln: + # Find the maintainers in kickstarts repo under + # maintainers.toml file and ping them in the ticket + # Find the branch in kickstarts repo + if self.config["ks_repo"]: + if 'rawhide' == msg['release_version'].lower(): + ks_branch = 'main' + elif 'epel' not in msg['release_name'].lower(): + ks_branch = 'f' + msg['release_version'] + ks_url = self.config["ks_repo"] + '/raw/' + ks_branch + '/f/maintainers.toml' + else: + ks_url = self.config["ks_repo"] + '/raw/main/f/maintainers.toml' kojitaskline, text = self.get_supporting_text(lines[x-1:]) content+=f'{text}\n' @@ -277,11 +309,17 @@ class Consumer(object): if self.config["ks_repo"]: # Ping the maintainers - maintainers, variant, sub_variant = self.get_maintainers(line, ks_url) or (None, None, None) + if not eln: + maintainers, variant, sub_variant = self.get_maintainers(line, ks_url) or (None, None, None) + else: + maintainers, variant, sub_variant = self.get_maintainers('eln', ks_url) or (None, None, None) if not maintainers: logger.info("No maintainers info available. The ticket will be filed without pinging any maintainers") else: - ping_line = f'Variant: {variant}, subvariant: {sub_variant} task failed. Pinging maintainers: ' + if not eln: + ping_line = f'Variant: {variant}, subvariant: {sub_variant} task failed. Pinging maintainers: ' + else: + ping_line = f'ELN compose has failed. Pinging maintainers: ' for maintainer in maintainers: ping_line += f'@{maintainer} ' content += ping_line + '\n\n' @@ -306,7 +344,10 @@ class Consumer(object): content+= f"Compose phase {p}: FAILED.\n" logger.debug(content) - labels = [msg.get("release_name"), msg.get("release_version"), msg.get("status")] + if not eln: + labels = [msg.get("release_name"), msg.get("release_version"), msg.get("status")] + else: + labels = ['eln'] logger.debug(f"Adding Labels {labels}") if self.token: @@ -341,8 +382,15 @@ if __name__ == '__main__': body = data['msg'] # Create a Message and then call the Consumer() - m = fedora_messaging.api.Message( - topic = 'org.fedoraproject.prod.pungi.compose.status.change', - body = body) - c = Consumer() - c.__call__(m) + if "ELN" in body['compose']['pungi_compose_id']: + m = fedora_messaging.api.Message( + topic = 'org.fedoraproject.prod.odcs.compose.state-changed', + body = body) + c = Consumer() + c.__call__(m, True) + else: + m = fedora_messaging.api.Message( + topic = 'org.fedoraproject.prod.pungi.compose.status.change', + body = body) + c = Consumer() + c.__call__(m)