From e2f2b99149a3c7eec5551a3cdfe09959f5d0b4a6 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Nov 28 2022 20:04:54 +0000 Subject: [PATCH 1/3] Handle non EL releases --- diff --git a/package_updates/utils/package.py b/package_updates/utils/package.py index 187d2ff..39ab394 100644 --- a/package_updates/utils/package.py +++ b/package_updates/utils/package.py @@ -14,8 +14,12 @@ class Package: Returns: String with centos version. """ - - ext = re.search('el[0-9]+', self.release).group() # '1.hs.el8' -> el8 - distro_num = re.search('[0-9]+', ext).group() # el8 -> 8 + + ext = re.search('el[0-9]+', self.release) # '1.hs.el8' -> el8 + # Ignore if there is not 'el' found in release + if not ext: + return + el = ext.group() # 'el8' + distro_num = re.search('[0-9]+', el).group() # el8 -> 8 distro = f'c{distro_num}s' return distro \ No newline at end of file From 8ead4708804e9159d49fc99f1dd1de1529812b9c Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Dec 01 2022 00:49:48 +0000 Subject: [PATCH 2/3] Added logging --- diff --git a/README.md b/README.md index 001931c..39569da 100755 --- a/README.md +++ b/README.md @@ -10,12 +10,13 @@ podman build -t package-updates . podman run -it \ --mount type=bind,src=$HOME/.centos-server-ca.cert,dst=/app/.centos-server-ca.cert,ro=true,relabel=private \ --mount type=bind,src=$HOME/.centos.cert,dst=/app/.centos.cert,ro=true,relabel=private \ - --mount type=bind,source=config.json,dst=/app/package-updates/config.json \ + --mount type=bind,source=$(pwd)config.json,dst=/app/package-updates/config.json \ -e PAGURE_API_KEY= \ -e CAFILE=/app/.centos-server-ca.cert \ -e CERT=/app/.centos.cert \ -e KEY=/app/.centos.cert \ - -e CONFIG_PATH=/app/package-updates/config.json \ + -e CONFIG_PATH=/app/package-updates/config.json \ + -e FEDORA_MESSAGING_CONF=/app/package-updates/fedora.toml \ -e RUN_MODE=MQTT \ package-updates ``` @@ -28,7 +29,7 @@ podman build -t package-updates . podman run -it \ --mount type=bind,src=$HOME/.centos-server-ca.cert,dst=/app/.centos-server-ca.cert,ro=true,relabel=private \ --mount type=bind,src=$HOME/.centos.cert,dst=/app/.centos.cert,ro=true,relabel=private \ - --mount type=bind,source=config.json,dst=/app/package-updates/config.json \ + --mount type=bind,source=$(pwd)/config.json,dst=/app/package-updates/config.json \ -e PAGURE_API_KEY= \ -e CAFILE=/app/.centos-server-ca.cert \ -e CERT=/app/.centos.cert \ @@ -73,7 +74,7 @@ Config.json example *The tags field is prioritized over the keyword field, if you provide both you will only get package updates from the list of tags specified.* -*For AMQP configuration use the fedora.toml file* +*For AMQP and logging configuration use the fedora.toml file* ### Enviroment variables diff --git a/fedora.toml b/fedora.toml index 1c23ff1..64178a2 100644 --- a/fedora.toml +++ b/fedora.toml @@ -61,7 +61,8 @@ version = 1 disable_existing_loggers = true [log_config.formatters.simple] -format = "[%(levelname)s %(name)s] %(message)s" +format = "[%(asctime)s %(levelname)s %(name)s] %(message)s" +datefmt = "%Y-%m-%d %H:%M:%S" [log_config.handlers.console] class = "logging.StreamHandler" @@ -86,10 +87,10 @@ handlers = ["console"] # If your consumer sets up a logger, you must add a configuration for it # here in order for the messages to show up. e.g. if it set up a logger # called 'example_printer', you could do: -#[log_config.loggers.example_printer] -#level = "INFO" -#propagate = false -#handlers = ["console"] +[log_config.loggers.package_updates] +level = "INFO" +propagate = false +handlers = ["console"] [log_config.root] level = "ERROR" diff --git a/package_updates/amqp/amqp.py b/package_updates/amqp/amqp.py index 2428401..e8c3600 100644 --- a/package_updates/amqp/amqp.py +++ b/package_updates/amqp/amqp.py @@ -1,6 +1,7 @@ +import logging import sqlite3 -from fedora_messaging import api, config +from fedora_messaging import api from twisted.internet import reactor from .package_build import PackageBuild @@ -10,7 +11,7 @@ from utils.db import BuildTargetDB, PackageDB, TagDB, get_connection from settings.constants import CBS_BUILD_TOPIC, CBS_URL, KOJIHUB_BUILD_TOPIC -config.conf.setup_logging() +_log = logging.getLogger('package_updates') class AMQP: @@ -42,7 +43,7 @@ class AMQP: new_issue = None session = helpers.get_koji_session(CBS_URL) latest_cbs_build = session.listTagged(saved_package['tag_id'], latest=True, package=package_build.name)[0]['nvr'] - print(f'upstream version: {package_build.package} | current version: {latest_cbs_build}') + _log.info(f'upstream version: {package_build.package} | current version: {latest_cbs_build}') if helpers.compare_versions(package_build.package, latest_cbs_build) == 1: conn = get_connection() tag_db = TagDB(conn) @@ -59,7 +60,7 @@ class AMQP: package build from cbs is complete. """ - print(str(msg)) + _log.info(str(msg)) topic, body = msg.topic, msg.body build = PackageBuild(body['name'], body['version'], body['release'], body['build_id'], body['new'], body['request'][0], body['request'][1]) diff --git a/package_updates/main.py b/package_updates/main.py index 35acbcb..4094326 100644 --- a/package_updates/main.py +++ b/package_updates/main.py @@ -1,8 +1,11 @@ #!/usr/bin/python +import logging import sys import time +from fedora_messaging import config + from amqp.amqp import AMQP from amqp.package_build import PackageBuild from mqtt.mqtt import MQTT @@ -10,6 +13,8 @@ from mqtt.package_git_release import PackageGitRelease from utils import helpers, db from settings.constants import CBS_URL, CBS_TAGS, CBS_TAGS_KEYWORD, C8S_GIT_BRANCH, C9S_PENDING, KOJIHUB_URL, RUN_MODE +config.conf.setup_logging() +_log = logging.getLogger('package_updates') def setup(conn): """Gets packages, creates database and sees if a package @@ -61,7 +66,7 @@ def setup(conn): def main(): - print('running...') + _log.info('running...') try: conn = db.get_connection() setup(conn) diff --git a/package_updates/mqtt/mqtt.py b/package_updates/mqtt/mqtt.py index 439a95e..0abedf4 100644 --- a/package_updates/mqtt/mqtt.py +++ b/package_updates/mqtt/mqtt.py @@ -1,4 +1,5 @@ import json +import logging import sqlite3 import paho.mqtt.client as mqtt @@ -10,6 +11,9 @@ from .package_git_release import PackageGitRelease from settings.constants import C8S_GIT_BRANCH, CBS_URL, CAFILE, CERT, KEY, MQTT_BROKER, MQTT_PORT, MQTT_TOPIC +_log = logging.getLogger('package_updates') + + class MQTT: """Class for MQTT run mode. """ @@ -23,9 +27,9 @@ class MQTT: """ if rc == 0: - print(f'Connected to MQTT Broker') + _log.info(f'Connected to MQTT Broker') else: - print(f'Failed to connect, return code {rc}') + _log.error(f'Failed to connect, return code {rc}') client.subscribe(MQTT_TOPIC) @@ -35,7 +39,7 @@ class MQTT: """ str_payload = msg.payload.decode('utf-8') - print(msg.topic + " " + str_payload) + _log.info(f'TOPIC: {msg.topic} BODY: {str_payload}') payload = json.loads(str_payload) nvr = helpers.filter_from_tag(payload['tag'], payload['repo']['name']) invr = helpers.split_package(nvr) @@ -58,7 +62,7 @@ class MQTT: cbs_package = session.listTagged(saved_package['tag_id'] , latest=True, package=package_release.name)[0]['nvr'] upstream_version = helpers.filter_from_tag(package_release.git_tag, package_release.name) - print(f'upstream version: {upstream_version} | current version: {cbs_package}') + _log.info(f'upstream version: {upstream_version} | current version: {cbs_package}') if helpers.compare_versions(upstream_version, cbs_package) == 1: conn = get_connection() tag_db = TagDB(conn) diff --git a/package_updates/utils/helpers.py b/package_updates/utils/helpers.py index 70996ce..d6fb62e 100755 --- a/package_updates/utils/helpers.py +++ b/package_updates/utils/helpers.py @@ -1,3 +1,4 @@ +import logging import re from urllib.parse import urljoin import rpm @@ -10,6 +11,9 @@ from settings.constants import GIT_CENTOS_API_URL, PAGURE_REPO_API_URL, PAGURE_A from .package import Package +_log = logging.getLogger('package_updates') + + def get_koji_session(url: str) -> koji.ClientSession: """Connects to the koji build system. @@ -38,10 +42,10 @@ def get_tagged_packages(session: koji.ClientSession, tag: int) -> list: try: packages = session.listPackages(tagID=tag) if not packages: - print(f"There is no available packages for {tag}") + _log.info(f"There is no available packages for {tag}") return except koji.GenericError as err: - print(err) + _log.exception(err) return packages @@ -243,7 +247,7 @@ def get_git_pkg_version(package: str, branch: str) -> tuple: res = requests.get(url=url, params=params) res.raise_for_status() except requests.exceptions.RequestException as err: - print(err) + _log.exception(err) return (None, None) tags = res.json()['tags'] @@ -315,22 +319,22 @@ def create_ticket(issue: dict) -> dict: url = urljoin(PAGURE_REPO_API_URL, 'new_issue') token = PAGURE_API_KEY headers = {'Authorization': f'access_token {token}'} - print('Creating ticket on pagure.io...') + _log.info('Creating ticket on pagure.io...') try: res = requests.post(url=url, data=issue, headers=headers) res.raise_for_status() except requests.exceptions.HTTPError as errh: if res.status_code == 401: - print(errh) + _log.exception(errh) raise SystemExit(f'{errh}\nMake sure you have the correct API token') - print(errh) + _log.exception(errh) return except requests.exceptions.RequestException as err: - print(err) + _log.exception(err) return - print('ticket created') + _log.info('ticket created') return res.json()['issue'] @@ -355,7 +359,7 @@ def is_issue_Open(tag: str) -> bool: res = requests.get(url=url, params=params) res.raise_for_status() except requests.exceptions.RequestException as err: - print(err) + _log.exception(err) return True total_issues = res.json()['total_issues'] @@ -385,7 +389,7 @@ def get_open_issue(tag: str) -> dict: res = requests.get(url=url, params=params) res.raise_for_status() except requests.exceptions.RequestException as err: - print(err) + _log.exception(err) return total_issues = res.json()['total_issues'] @@ -413,15 +417,15 @@ def comment_on_issue(issue_id: int, comment: dict): res.raise_for_status() except requests.exceptions.HTTPError as errh: if res.status_code == 401: - print(errh) + _log.exception(errh) raise SystemExit(f'{errh}\nMake sure you have the correct API token') - print(errh) + _log.exception(errh) return except requests.exceptions.RequestException as err: - print(err) + _log.exception(err) return - print(f'Comment added on issue {issue_id}') + _log.info(f'Comment added on issue {issue_id}') def get_issues(tags: list) -> list: @@ -444,7 +448,7 @@ def get_issues(tags: list) -> list: res = requests.get(url=url, params=params) res.raise_for_status() except requests.exceptions.RequestException as err: - print(err) + _log.exception(err) return None issues = res.json()['issues'] @@ -471,10 +475,10 @@ def close_issue(issue_id: int, close_status: str): res = requests.post(url=url, data=data, headers=headers) res.raise_for_status() except requests.exceptions.RequestException as err: - print(err) - print(res.json()) + _log.exception(err) + _log.exception(res.json()) return - print('issue closed') + _log.info(f'issue {issue_id} closed') def issue_filter(issues: list, package_name: str) -> dict: From 157a7e9907945c243a6a58ae496ded9efc34e7d9 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Dec 01 2022 02:54:09 +0000 Subject: [PATCH 3/3] Handle null request in amqp build body --- diff --git a/package_updates/amqp/amqp.py b/package_updates/amqp/amqp.py index e8c3600..feb144a 100644 --- a/package_updates/amqp/amqp.py +++ b/package_updates/amqp/amqp.py @@ -63,6 +63,11 @@ class AMQP: _log.info(str(msg)) topic, body = msg.topic, msg.body + # Ignore if the request field in body is null (needed to get build target and source) + if not body['request']: + _log.info("request field is null, skipping messasge") + return + build = PackageBuild(body['name'], body['version'], body['release'], body['build_id'], body['new'], body['request'][0], body['request'][1]) if build.is_build_complete() and topic == CBS_BUILD_TOPIC: diff --git a/tests/test_amqp_pub/publish.py b/tests/test_amqp_pub/publish.py index 2166508..a4dd3c3 100644 --- a/tests/test_amqp_pub/publish.py +++ b/tests/test_amqp_pub/publish.py @@ -1,3 +1,4 @@ +import time from fedora_messaging import api, message, config config.conf.setup_logging() @@ -45,9 +46,74 @@ msg2 = message.Message(topic=u'nice.message2', headers={u'niceness': u'very'}, u'version': u'4.16.1.3' }) -print("Select topic (1 or 2): ") -t = int(input()) -if t == 1: - api.publish(msg) -else: - api.publish(msg2) \ No newline at end of file +msg3 = message.Message(topic=u'nice.message2', headers={u'niceness': u'very'}, + body={ + u'attribute': u"state", + u'build_id': 27411, + u'epoch': None, + u'instance': 'primary', + u'name': u'gnutls', + u'new': 1, + u'old': None, + u'owner': u'centos-stream', + u'release': u'5.el8', + u'request': None, + u'task_id': None, + u'version': u'3.6.16' + }) + +msg4 = message.Message(topic=u'nice.message2', headers={u'niceness': u'very'}, + body={ + u'attribute': u"state", + u'build_id': 27394, + u'epoch': 0, + u'instance': 'primary', + u'name': u'CentOS-Stream-Container-Base', + u'new': 1, + u'old': 0, + u'owner': u'odcs/odcs.stream.rdu2.redhat.com', + u'release': u'20221130.0', + u'request': [ + u'CentOS-Stream-Container-Base', + '9', + [ + u'aarch64', + u'ppc64le', + u's390x', + u'x86_64' + ], + u'c9s-containers', + u'http://composefiles.stream.rdu2.redhat.com/odcs-2658/compose/BaseOS/$arch/os', + { + u'disk_size': u'10', + u'distro': u'Fedora-28', + u'factory_parameter': [ + [ + u'generate_icicle', + u'False' + ] + ], + u'format': [ + u'docker' + ], + u'kickstart': u'CentOS-Stream-9-container-base.ks', + u'ksurl': u'git+https://gitlab.com/redhat/centos-stream/release-engineering/kickstarts.git?#ba71edcfd5e9d0c4c100894f75e998165d79ce44', + u'ksversion': u'F28', + u'optional_arches': u'', + u'releaseu': u'20221130.0', + u'repou': [ + u'http://composefiles.stream.rdu2.redhat.com/odcs-2658/compose/BaseOS/$arch/os', + u'http://composefiles.stream.rdu2.redhat.com/odcs-2658/compose/AppStream/$arch/os' + ] + } + ], + u'task_id': 1636340, + u'version': u'9' + }) + + +msgs = [msg2, msg3, msg4, msg] + +for m in msgs: + time.sleep(5) + api.publish(m) \ No newline at end of file