From da77667a20aa56d1cf7ebf33eb97993b1fa9a492 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Nov 16 2022 18:42:48 +0000 Subject: [PATCH 1/7] Make global db connection --- diff --git a/package_updates/utils/db.py b/package_updates/utils/db.py index fbff923..5d2dcd2 100755 --- a/package_updates/utils/db.py +++ b/package_updates/utils/db.py @@ -1,6 +1,7 @@ import sqlite3 import re +__conn = None def get_connection() -> sqlite3.Connection: """Creates an in-memory sqlite connection. @@ -8,10 +9,12 @@ def get_connection() -> sqlite3.Connection: Returns: A sqlite connection with sqlite3.Row as the row factory. """ - - conn = sqlite3.connect("file:mem1?mode=memory&cache=shared", uri=True, check_same_thread=False) - conn.row_factory = sqlite3.Row - return conn + + global __conn + if not __conn: + __conn = sqlite3.connect("file:mem1?mode=memory&cache=shared", uri=True, check_same_thread=False) + __conn.row_factory = sqlite3.Row + return __conn class BuildTargetDB: From c47233dd5ad88e40ea9cce5d528d233af2cf7697 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Nov 16 2022 19:02:43 +0000 Subject: [PATCH 2/7] Add c9s koji tags & topic --- diff --git a/package_updates/settings/constants.py b/package_updates/settings/constants.py index ce1e731..7f1def5 100644 --- a/package_updates/settings/constants.py +++ b/package_updates/settings/constants.py @@ -11,6 +11,7 @@ with open(CONFIG_PATH) as f: CBS_URL = 'https://cbs.centos.org/kojihub' CBS_BUILD_URL = 'https://cbs.centos.org/koji/buildinfo?buildID=' KOJIHUB_URL = 'https://kojihub.stream.centos.org/kojihub' +KOJIHUB_BUILD_URL = 'https://kojihub.stream.centos.org/koji/buildinfo?buildID=' GIT_CENTOS_API_URL = 'https://git.centos.org/api/0/rpms/' GIT_CENTOS_URL = 'https://git.centos.org/rpms/' PAGURE_REPO_URL = config['pagure']['repo_url'] or 'https://localhost/foo/bar/' @@ -40,6 +41,7 @@ client_id = f'hyperscale-sig-{random.randint(0, 1000)}' # AMQP CBS_BUILD_TOPIC = 'org.centos.prod.cbs.buildsys.build.state.change' +KOJIHUB_BUILD_TOPIC = 'org.centos.prod.buildsys.build.state.change' # CBS tags @@ -47,5 +49,11 @@ CBS_TAGS = config['cbs']['tags'] CBS_TAGS_KEYWORD = config['cbs']['tags_keyword'] +#c9s kojihub tags +C9S_PENDING = 1 # c9s-pending +C9S_GATE = 2 # c9s-gate +C9S_CANDIDATE = 3 # c9s-candidate + + # Run mode RUN_MODE = os.getenv('RUN_MODE', default='MQTT') \ No newline at end of file From d301654e33319786ebc0628e5a0addae3b892cef Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Nov 16 2022 19:20:40 +0000 Subject: [PATCH 3/7] Add koji build topic to fedora.toml & modify readme --- diff --git a/README.md b/README.md index e48727b..ebc2fda 100755 --- a/README.md +++ b/README.md @@ -8,14 +8,14 @@ git clone https://pagure.io/centos-sig-hyperscale/package-updates.git cd package-updates podman build -t package-updates . podman run -it \ - --mount type=bind,src=$HOME/.centos-server-ca.cert,dst=/home/app/.centos-server-ca.cert,ro=true,relabel=private \ - --mount type=bind,src=$HOME/.centos.cert,dst=/home/app/.centos.cert,ro=true,relabel=private \ - --mount type=bind,source=config.json,dst=/home/app/package-updates/config.json \ + --mount type=bind,src=$HOME/.centos-server-ca.cert,dst=/app/.centos-server-ca.cert,ro=true,relabel=private \ + --mount type=bind,src=$HOME/.centos.cert,dst=/app/.centos.cert,ro=true,relabel=private \ + --mount type=bind,source=config.json,dst=/app/package-updates/config.json \ -e PAGURE_API_KEY= \ - -e CAFILE=/home/app/.centos-server-ca.cert \ - -e CERT=/home/app/.centos.cert \ - -e KEY=/home/app/.centos.cert \ - -e CONFIG_PATH=/home/app/package-updates/config.json \ + -e CAFILE=/app/.centos-server-ca.cert \ + -e CERT=/app/.centos.cert \ + -e KEY=/app/.centos.cert \ + -e CONFIG_PATH=/app/package-updates/config.json \ -e RUN_MODE=MQTT \ package-updates ``` @@ -26,17 +26,18 @@ git clone https://pagure.io/centos-sig-hyperscale/package-updates.git cd package-updates podman build -t package-updates . podman run -it \ ---mount type=bind,src=$HOME/.centos-server-ca.cert,dst=/home/app/.centos-server-ca.cert,ro=true,relabel=private \ ---mount type=bind,src=$HOME/.centos.cert,dst=/home/app/.centos.cert,ro=true,relabel=private \ ---mount type=bind,source=config.json,dst=/home/app/package-updates/config.json \ --e PAGURE_API_KEY= \ --e CAFILE=/home/app/.centos-server-ca.cert \ --e CERT=/home/app/.centos.cert \ --e KEY=/home/app/.centos.cert \ --e CONFIG_PATH=/home/app/package-updates/config.json \ --e FEDORA_MESSAGING_CONF=/home/app/package-updates/fedora.toml \ --e RUN_MODE=AMQP \ -package-updates + --mount type=bind,src=$HOME/.centos-server-ca.cert,dst=/app/.centos-server-ca.cert,ro=true,relabel=private \ + --mount type=bind,src=$HOME/.centos.cert,dst=/app/.centos.cert,ro=true,relabel=private \ + --mount type=bind,source=config.json,dst=/app/package-updates/config.json \ + --mount type=bind,source=fedora.toml,dst=/app/package-updates/fedora.toml,ro=true,relabel=private \ + -e PAGURE_API_KEY= \ + -e CAFILE=/app/.centos-server-ca.cert \ + -e CERT=/app/.centos.cert \ + -e KEY=/app/.centos.cert \ + -e CONFIG_PATH=/app/package-updates/config.json \ + -e FEDORA_MESSAGING_CONF=/app/package-updates/fedora.toml \ + -e RUN_MODE=AMQP \ + package-updates ``` ### Configuration diff --git a/fedora.toml b/fedora.toml index 75e5694..1c23ff1 100644 --- a/fedora.toml +++ b/fedora.toml @@ -47,7 +47,7 @@ arguments = {} [[bindings]] # queue = "00000000-0000-0000-0000-000000000000" exchange = "amq.topic" -routing_keys = ["org.centos.prod.cbs.buildsys.build.state.change"] # Set this to the specific topics you are interested in. +routing_keys = ["org.centos.prod.cbs.buildsys.build.state.change", "org.centos.prod.buildsys.build.state.change"] # Set this to the specific topics you are interested in. [consumer_config] example_key = "for my consumer" From f6d032c5dc6146a1decffb031028dba150f7303a Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Nov 16 2022 19:27:48 +0000 Subject: [PATCH 4/7] Add c9s package updates % fix package updates logic --- diff --git a/package_updates/amqp/amqp.py b/package_updates/amqp/amqp.py index 3da01af..a15defb 100644 --- a/package_updates/amqp/amqp.py +++ b/package_updates/amqp/amqp.py @@ -5,8 +5,9 @@ from twisted.internet import reactor from .package_build import PackageBuild from utils import helpers -from utils.db import BuildTargetDB -from settings.constants import CBS_BUILD_TOPIC +from utils.helpers import duplicateValidator +from utils.db import BuildTargetDB, PackageDB, TagDB, get_connection +from settings.constants import CBS_BUILD_TOPIC, CBS_URL, KOJIHUB_BUILD_TOPIC config.conf.setup_logging() @@ -26,16 +27,34 @@ class AMQP: centos_version = helpers.get_centos_version(build.release) issue = helpers.get_open_issue(f'{build.name}.{centos_version}') if issue: - build_url = build.get_build_url() + build_url = build.get_build_url('cbs') commit_url = build.get_commit_url() - comment = { - 'comment': f"""Build: {build_url} - Commit: {commit_url}""" - } + comment = helpers.fixed_status_comment(build_url, commit_url) helpers.comment_on_issue(issue['id'], comment) helpers.close_issue(issue['id'], 'Fixed') - def __on_message(self, msg): + @staticmethod + @duplicateValidator + def package_updater(package_build: PackageBuild, saved_package: dict): + """Creates an issue if there is a package update. + """ + + new_issue = None + session = helpers.get_koji_session(CBS_URL) + latest_cbs_build = session.listTagged(saved_package['tag_id'], latest=True, package=package_build.name)[0]['nvr'] + print(f'upstream version: {package_build.package} | current version: {latest_cbs_build}') + if helpers.compare_versions(package_build.package, latest_cbs_build) == 1: + conn = get_connection() + tag_db = TagDB(conn) + tag_name = tag_db.select_tag(saved_package['tag_id'])['tag_name'] + centos_version = package_build.get_centos_version() + commit_url = package_build.get_commit_url() + issue_conent = helpers.issue_content(package_build.name, centos_version, package_build.package, latest_cbs_build, commit_url, tag_name) + new_issue = helpers.create_ticket(issue_conent) + + return new_issue + + def __on_message(self, msg) -> None: """Callback. Called when a message is received by fedora messaging, checks if a package build from cbs is complete and if the build target is saved in the DB. """ @@ -43,15 +62,20 @@ class AMQP: print(str(msg)) topic, body = msg.topic, msg.body - build = PackageBuild(body) - build_target_db = BuildTargetDB(self.db_conn) - saved_build_target = build_target_db.select_build_target(build.build_target) + build = PackageBuild(body['name'], body['version'], body['release'], body['build_id'], body['new'], body['request'][0], body['request'][1]) - if build.is_build_complete() and saved_build_target: - if topic == CBS_BUILD_TOPIC: + if build.is_build_complete() and topic == CBS_BUILD_TOPIC: + build_target_db = BuildTargetDB(self.db_conn) + saved_build_target = build_target_db.select_build_target(build.build_target) + if saved_build_target: self.__issue_closer(build) + elif build.is_build_complete() and topic == KOJIHUB_BUILD_TOPIC: + pkg_db = PackageDB(self.db_conn) + pkg = pkg_db.select_package(build.name, build.get_centos_version()) + if pkg: + AMQP.package_updater(build, pkg) - def listen_on_amqp(self): + def listen_on_amqp(self) -> None: """Connects to fedora messaging AMQP broker and keeps listening """ diff --git a/package_updates/amqp/package_build.py b/package_updates/amqp/package_build.py index f155e6f..90b843d 100644 --- a/package_updates/amqp/package_build.py +++ b/package_updates/amqp/package_build.py @@ -1,37 +1,51 @@ import re -from settings.constants import CBS_BUILD_URL +from utils.package import Package +from settings.constants import CBS_BUILD_URL, KOJIHUB_BUILD_URL -class PackageBuild: +class PackageBuild(Package): """Represents a package build entity from the AMQP #buildsys.build.state.change topics payload. Gets only necessary values from the payload. """ - def __init__(self, body_msg: dict) -> None: - self.name = body_msg['name'] - self.version = body_msg['version'] - self.release = body_msg['release'] - self.build_id = body_msg['build_id'] - self.build_status = body_msg['new'] - self.source = body_msg['request'][0] - self.build_target = body_msg['request'][1] - self.package = ('%s-%s-%s' % (self.name, self.version, self.release)) + def __init__(self, name, version, release, build_id, build_status, source, build_target) -> None: + self.build_id = build_id + self.build_status = build_status + self.source = source + self.build_target = build_target + self.package = ('%s-%s-%s' % (name, version, release)) + super().__init__(name, version, release) def is_build_complete(self) -> bool: """Checks if a package build from koji is complete. + + Returns: + True if build status is complete. False otherwise. """ return self.build_status == 1 - def get_build_url(self) -> str: + def get_build_url(self, koji_instance: str) -> str: """Generates a url for a koji package build. + + Args: + koji_instance: String of koji instance. e.g. 'cbs', 'kojihub' + + Returns: + String of koji/cbs build url. """ - return f'{CBS_BUILD_URL}{self.build_id}' + if koji_instance == 'cbs': + return f'{CBS_BUILD_URL}{self.build_id}' + + return f'{KOJIHUB_BUILD_URL}{self.build_id}' def get_commit_url(self) -> str: """Gets the commit url from the #buildsys.build.state.change topic payload. + + Returns: + String with build source commit url. """ url = re.findall(r'(https?://\S+)', self.source) diff --git a/package_updates/main.py b/package_updates/main.py index 7fa0bc8..35acbcb 100644 --- a/package_updates/main.py +++ b/package_updates/main.py @@ -2,12 +2,13 @@ import sys import time -from urllib.parse import urljoin from amqp.amqp import AMQP +from amqp.package_build import PackageBuild from mqtt.mqtt import MQTT +from mqtt.package_git_release import PackageGitRelease from utils import helpers, db -from settings.constants import CBS_URL, CBS_TAGS, CBS_TAGS_KEYWORD, C8S_GIT_BRANCH, RUN_MODE, PAGURE_REPO_URL +from settings.constants import CBS_URL, CBS_TAGS, CBS_TAGS_KEYWORD, C8S_GIT_BRANCH, C9S_PENDING, KOJIHUB_URL, RUN_MODE def setup(conn): @@ -39,42 +40,25 @@ def setup(conn): issue_db = db.IssueDB(conn) issue_db.create_issue_table() - + saved_packages = package_db.select_packages() for package in saved_packages: - package_name = package['name'] - hs_version = ('%s-%s-%s' % (package['name'], package['version'], package['release'])) - cbs_tag_id = package['tag_id'] + if package['distro_version'] == C8S_GIT_BRANCH: + upstream_version, commit = helpers.get_git_pkg_version(package['name'], C8S_GIT_BRANCH) + if upstream_version == None: + continue + nvr = helpers.split_package(upstream_version) + package_release = PackageGitRelease(nvr[0], nvr[1], nvr[2], upstream_version, commit) + MQTT.package_updater(package_release, package) + else: + koji_session = helpers.get_koji_session(KOJIHUB_URL) + latest_tagged_build = koji_session.listTagged(C9S_PENDING, latest=True, package=package['name']) + if len(latest_tagged_build) == 0: + continue + latest_koji_build = koji_session.getBuild(latest_tagged_build[0]['build_id']) + package_build = PackageBuild(latest_koji_build['name'], latest_koji_build['version'], latest_koji_build['release'], latest_koji_build['build_id'], latest_koji_build['state'], latest_koji_build['source'], package) + AMQP.package_updater(package_build, package) - upstream_pkg_version, commit = helpers.get_git_pkg_version(package_name, C8S_GIT_BRANCH) - if upstream_pkg_version == None: - continue - - print(f'centos version: {upstream_pkg_version} | hs version: {hs_version}') - if helpers.compare_versions(upstream_pkg_version, hs_version) == 1: - centos_version = helpers.get_centos_version(upstream_pkg_version) - hs_tag_name = tag_db.select_tag(cbs_tag_id)['tag_name'] - issues = helpers.get_issues([package_name]) #gets all issues tagged with current package name - issue = helpers.issue_filter(issues, package_name) #gets the issue that is tagged with package name and package version - if issue: - version_tag = issue['version_tag'] - if upstream_pkg_version != version_tag: - new_issue = helpers.create_ticket(package_name, upstream_pkg_version, hs_version, hs_tag_name, commit, centos_version) - new_issue_url = urljoin(PAGURE_REPO_URL, f"issue/{new_issue['id']}") - comment = { - 'comment': f"""{upstream_pkg_version} is available and this issue still open. - New issue has been created for the newer version. - URL: {new_issue_url}""" - } - helpers.comment_on_issue(issue['issue_id'], comment) - helpers.close_issue(issue['issue_id'], 'Invalid') - issue_db.insert_issue(new_issue['id'], package['package_id']) - else: - issue_db.insert_issue(issue['issue_id'], package['package_id']) - else: - new_issue = helpers.create_ticket(package_name, upstream_pkg_version, hs_version, hs_tag_name, commit, centos_version) - issue_db.insert_issue(new_issue['id'], package['package_id']) - def main(): print('running...') diff --git a/package_updates/mqtt/mqtt.py b/package_updates/mqtt/mqtt.py index 82be156..439a95e 100644 --- a/package_updates/mqtt/mqtt.py +++ b/package_updates/mqtt/mqtt.py @@ -1,13 +1,13 @@ import json import sqlite3 -from urllib.parse import urljoin import paho.mqtt.client as mqtt from utils import helpers -from utils.db import PackageDB, IssueDB, TagDB +from utils.helpers import duplicateValidator +from utils.db import PackageDB, TagDB, get_connection from .package_git_release import PackageGitRelease -from settings.constants import C8S_GIT_BRANCH, CBS_URL, CAFILE, CERT, KEY, MQTT_BROKER, MQTT_PORT, PAGURE_REPO_URL, MQTT_TOPIC +from settings.constants import C8S_GIT_BRANCH, CBS_URL, CAFILE, CERT, KEY, MQTT_BROKER, MQTT_PORT, MQTT_TOPIC class MQTT: @@ -17,7 +17,7 @@ class MQTT: def __init__(self, db_conn: sqlite3.Connection) -> None: self.db_conn = db_conn - def __on_connect(self, client, user_data, flags, rc): + def __on_connect(self, client, user_data, flags, rc) -> None: """Callback, establishes a connection with the mqtt server and subscribes to a specific topic. """ @@ -29,7 +29,7 @@ class MQTT: client.subscribe(MQTT_TOPIC) - def __on_message(self, client, user_data, msg): + def __on_message(self, client, user_data, msg) -> None: """Callback, when a message from mqtt is received deserializes the payload and checks if it's a new package version to create an issue. """ @@ -37,50 +37,40 @@ class MQTT: str_payload = msg.payload.decode('utf-8') print(msg.topic + " " + str_payload) payload = json.loads(str_payload) - - package_release = PackageGitRelease(payload) + nvr = helpers.filter_from_tag(payload['tag'], payload['repo']['name']) + invr = helpers.split_package(nvr) + package_release = PackageGitRelease(payload['repo']['name'], invr[1], invr[2], payload['tag'], payload['rev']) db_conn = user_data['db_conn'] pkg_db = PackageDB(db_conn) - saved_hs_package = pkg_db.select_package(package_release.name, C8S_GIT_BRANCH) - if saved_hs_package and C8S_GIT_BRANCH in package_release.git_tag: - self.__package_updater(package_release, saved_hs_package) + saved_package = pkg_db.select_package(package_release.name, C8S_GIT_BRANCH) + if saved_package and C8S_GIT_BRANCH in package_release.git_tag: + MQTT.package_updater(package_release, saved_package) - def __package_updater(self, package_release: PackageGitRelease, saved_hs_package): - """Creates in issue for a package update. + @staticmethod + @duplicateValidator + def package_updater(package_release: PackageGitRelease, saved_package: dict): + """Creates an issue if there is a package update. """ - pkg_name = package_release.name - - issue_db = IssueDB(self.db_conn) - tag_db = TagDB(self.db_conn) + new_issue = None session = helpers.get_koji_session(CBS_URL) - hs_tag_id = saved_hs_package['tag_id'] - cbs_hs_package = session.listTagged(hs_tag_id, latest=True, package=pkg_name) - hs_version = cbs_hs_package[0]['nvr'] - centos_version = helpers.filter_from_tag(package_release.git_tag, pkg_name) + cbs_package = session.listTagged(saved_package['tag_id'] , latest=True, package=package_release.name)[0]['nvr'] + upstream_version = helpers.filter_from_tag(package_release.git_tag, package_release.name) + + print(f'upstream version: {upstream_version} | current version: {cbs_package}') + if helpers.compare_versions(upstream_version, cbs_package) == 1: + conn = get_connection() + tag_db = TagDB(conn) + tag_name = tag_db.select_tag(saved_package['tag_id'])['tag_name'] + centos_version = package_release.get_centos_version() + commit_url = package_release.get_commit_url() + issue_conent = helpers.issue_content(package_release.name, centos_version, upstream_version, cbs_package, commit_url, tag_name) + new_issue = helpers.create_ticket(issue_conent) - print(f'new version: {centos_version} | current version: {hs_version}') - if helpers.compare_versions(centos_version, hs_version) == 1: - issue = issue_db.select_issue(saved_hs_package['package_id']) - hs_tag_name = tag_db.select_tag(hs_tag_id)['tag_name'] - is_open = helpers.is_issue_Open(f'{pkg_name}.c8s') - new_issue = helpers.create_ticket(pkg_name, centos_version, hs_version, hs_tag_name, package_release.commit, C8S_GIT_BRANCH) - if issue: - issue_db.update_issue_row(new_issue['id'], saved_hs_package['package_id']) - if is_open: - new_issue_url = urljoin(PAGURE_REPO_URL, f"issue/{new_issue['id']}") - comment = { - 'comment': f"""{centos_version} is available and this issue still open. - New issue has been created for the newer version. - URL: {new_issue_url}""" - } - helpers.comment_on_issue(issue['issue_id'], comment) - helpers.close_issue(issue['issue_id'], 'Invalid') - else: - issue_db.insert_issue(new_issue['id'], saved_hs_package['package_id']) + return new_issue - def listen_on_updates(self): + def listen_on_updates(self) -> None: """Sets up the mqtt client, starts the loop, and listens on git.centos.org notifications for tag creation. """ diff --git a/package_updates/mqtt/package_git_release.py b/package_updates/mqtt/package_git_release.py index 6fe0328..4019844 100644 --- a/package_updates/mqtt/package_git_release.py +++ b/package_updates/mqtt/package_git_release.py @@ -1,11 +1,25 @@ +from urllib.parse import urljoin -class PackageGitRelease: +from settings.constants import GIT_CENTOS_URL +from utils.package import Package + + +class PackageGitRelease(Package): """Represents a package git release tag from the MQTT git.centos.org/git.tag.creation topic payload. Gets only necessary values from the payload. """ - def __init__(self, payload: dict) -> None: - self.name = payload['repo']['name'] - self.git_tag = payload['tag'] - self.commit = payload['rev'] + def __init__(self, name, version, release, git_tag, commit) -> None: + self.git_tag = git_tag + self.commit = commit + super().__init__(name, version, release) + + def get_commit_url(self) -> str: + """Generates a commit rul from commit id. + + Returns: + String with git.centos.org commit url. + """ + + return urljoin(GIT_CENTOS_URL, f'{self.name}/tree/{self.commit}') diff --git a/package_updates/utils/helpers.py b/package_updates/utils/helpers.py index ccdcba2..c192ca9 100755 --- a/package_updates/utils/helpers.py +++ b/package_updates/utils/helpers.py @@ -4,8 +4,10 @@ import rpm import hawkey import koji import requests +import functools -from settings.constants import GIT_CENTOS_API_URL, GIT_CENTOS_URL, PAGURE_REPO_API_URL, PAGURE_API_KEY +from settings.constants import GIT_CENTOS_API_URL, PAGURE_REPO_API_URL, PAGURE_API_KEY, PAGURE_REPO_URL +from .package import Package def get_koji_session(url: str) -> koji.ClientSession: @@ -274,26 +276,42 @@ def get_centos_version(package: str) -> str: return distro -def create_ticket(pkg: str, upstream_version: str, current_version: str, cbs_tag_name: str, commit: str, distro: str): - """Creates a ticket on paguire.io repo. - +def issue_content(package_name: str, distro: str, upstream_nvr: str, current_nvr: str, commit_url: str, cbs_tag: str) -> dict: + """Creates a dictionary for issue content. The dict conatins title, issue_content and tag. + Args: - pkg: Package name. - upstream_version: New centos upstream version. - current_version: Current package version. - cbs_tag_name: Tag name from package. - commit: Commit id from source. - distro: Centos version + issue: Dictionary of issue containing title, issue_content and tag + package_name: String of package name. + distro: String of Centos version. E.g. 'c8s' + upstream_nvr: String of NVR package. E.g. dnf-4.7.0-8.el8 + current_nvr: String of NVR package. E.g. dnf-4.7.0-7.1.hs.el8 + commit_url: String of upstream package commit url. + cbs_tag: String of package tag name. + + Returns: + Dictionary with issue content. """ - - commit_url = urljoin(GIT_CENTOS_URL, f'{pkg}/tree/{commit}') + issue = { - 'title': f'{upstream_version} is available', - 'issue_content': f"""Latest upstream release: {upstream_version} - Current version/release: {current_version} + 'title': f'{upstream_nvr} is available', + 'issue_content': f"""Latest upstream release: {upstream_nvr} + Current version/release: {current_nvr} URL: {commit_url}""", - 'tag': f'{cbs_tag_name},{upstream_version},{pkg},{pkg}.{distro}' + 'tag': f'{cbs_tag},{upstream_nvr},{package_name},{package_name}.{distro}' } + return issue + + +def create_ticket(issue: dict) -> dict: + """Creates a ticket on paguire.io repo. + + Args: + issue: Dictionary of issue containing title, issue_content and tag + + Returns: + Dictionary with new ticket content. + """ + url = urljoin(PAGURE_REPO_API_URL, 'new_issue') token = PAGURE_API_KEY headers = {'Authorization': f'access_token {token}'} @@ -479,4 +497,54 @@ def issue_filter(issues: list, package_name: str) -> dict: if split_package(tag) and get_epel(tag): version_tag = tag return {'issue_id': issue['id'], 'package_name': package_name, 'version_tag': version_tag} - return None \ No newline at end of file + return None + + +def fixed_status_comment(build_url: str, commit_url: str) -> dict: + comment = { + 'comment': f"""Build: {build_url} + Commit: {commit_url}""" + } + return comment + + +def invalid_status_comment(new_issue_id: int, package: str) -> dict: + """ + Args: + new_isue_id: Issue id to generate issue link + package: String of package nvr + """ + new_issue_url = urljoin(PAGURE_REPO_URL, f"issue/{new_issue_id}") + comment = { + 'comment': f"""{package} is available and this issue still open. + New issue has been created for the newer version. + URL: {new_issue_url}""" + } + return comment + + +def duplicateValidator(func): + """"Decorator. Make sure there is not duplicate issues for same package. + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + for arg in args: + if isinstance(arg, Package): + package = arg + issue = None + distro_version = package.get_centos_version() + curr_issue = get_open_issue(f"{package.name}.{distro_version}") + if curr_issue: + prev_issue = issue_filter([curr_issue], package.name) + splitted = split_package(prev_issue['version_tag']) + if ('%s-%s-%s' % (splitted[0], splitted[1], splitted[2])) != ('%s-%s-%s' % (package.name, package.version, package.release)): + issue = func(*args, **kwargs) + if issue: + comment = invalid_status_comment(issue['id'], package.name) + comment_on_issue(prev_issue['issue_id'], comment) + close_issue(prev_issue['issue_id'], 'Invalid') + else: + issue = func(*args, **kwargs) + return issue + return wrapper \ No newline at end of file diff --git a/package_updates/utils/package.py b/package_updates/utils/package.py new file mode 100644 index 0000000..187d2ff --- /dev/null +++ b/package_updates/utils/package.py @@ -0,0 +1,21 @@ +import re + + +class Package: + def __init__(self, name, version, release) -> None: + self.name = name + self.version = version + self.release = release + + def get_centos_version(self) -> str: + """Gets centos distro version from package release + Returns string of centos version. e.g. 'c8s' + + Returns: + String with centos version. + """ + + ext = re.search('el[0-9]+', self.release).group() # '1.hs.el8' -> el8 + distro_num = re.search('[0-9]+', ext).group() # el8 -> 8 + distro = f'c{distro_num}s' + return distro \ No newline at end of file diff --git a/tests/test_amqp_pub/publish.py b/tests/test_amqp_pub/publish.py index f4fcd68..2166508 100644 --- a/tests/test_amqp_pub/publish.py +++ b/tests/test_amqp_pub/publish.py @@ -24,4 +24,30 @@ msg = message.Message(topic=u'nice.message', headers={u'niceness': u'very'}, u'task_id': 2940860, u'version': u'4.7.0' }) -api.publish(msg) \ No newline at end of file + +msg2 = message.Message(topic=u'nice.message2', headers={u'niceness': u'very'}, + body={ + u'attribute': u"state", + u'build_id': 40526, + u'epoch': 1, + u'instance': 'primary', + u'name': u'rpm', + u'new': 1, + u'old': 0, + u'owner': u'kmodsbot', + u'release': u'20.el9', + u'request': [ + u'git+https://gitlab.com/redhat/centos-stream/rpms/rpm.git', + u'c9s-candidate', + {} + ], + u'task_id': 1483326, + u'version': u'4.16.1.3' + }) + +print("Select topic (1 or 2): ") +t = int(input()) +if t == 1: + api.publish(msg) +else: + api.publish(msg2) \ No newline at end of file From 401bf216da2cf79f005c73e2b9799ca90bb136c6 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Nov 16 2022 19:34:25 +0000 Subject: [PATCH 5/7] Change amqp instructions --- diff --git a/README.md b/README.md index ebc2fda..3f7eda7 100755 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ podman run -it \ --mount type=bind,src=$HOME/.centos-server-ca.cert,dst=/app/.centos-server-ca.cert,ro=true,relabel=private \ --mount type=bind,src=$HOME/.centos.cert,dst=/app/.centos.cert,ro=true,relabel=private \ --mount type=bind,source=config.json,dst=/app/package-updates/config.json \ - --mount type=bind,source=fedora.toml,dst=/app/package-updates/fedora.toml,ro=true,relabel=private \ -e PAGURE_API_KEY= \ -e CAFILE=/app/.centos-server-ca.cert \ -e CERT=/app/.centos.cert \ From b7e33b302b9035255e5128a0b05c4c668866d3ad Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Nov 16 2022 21:47:45 +0000 Subject: [PATCH 6/7] Change invalid status comment info --- diff --git a/package_updates/utils/helpers.py b/package_updates/utils/helpers.py index c192ca9..165a939 100755 --- a/package_updates/utils/helpers.py +++ b/package_updates/utils/helpers.py @@ -541,7 +541,7 @@ def duplicateValidator(func): if ('%s-%s-%s' % (splitted[0], splitted[1], splitted[2])) != ('%s-%s-%s' % (package.name, package.version, package.release)): issue = func(*args, **kwargs) if issue: - comment = invalid_status_comment(issue['id'], package.name) + comment = invalid_status_comment(issue['id'], '%s-%s-%s' % (package.name, package.version, package.release)) comment_on_issue(prev_issue['issue_id'], comment) close_issue(prev_issue['issue_id'], 'Invalid') else: From c1867d27434f9802a9f829fea8e697fb3d459aad Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Nov 17 2022 19:40:12 +0000 Subject: [PATCH 7/7] Fix typos --- diff --git a/README.md b/README.md index 3f7eda7..001931c 100755 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ Config.json example } ``` -*The tags filed is prioritized over the keyword field, if you provide both you will only get package updates from the list of tags specified.* +*The tags field is prioritized over the keyword field, if you provide both you will only get package updates from the list of tags specified.* *For AMQP configuration use the fedora.toml file* diff --git a/package_updates/amqp/amqp.py b/package_updates/amqp/amqp.py index a15defb..2428401 100644 --- a/package_updates/amqp/amqp.py +++ b/package_updates/amqp/amqp.py @@ -56,7 +56,7 @@ class AMQP: def __on_message(self, msg) -> None: """Callback. Called when a message is received by fedora messaging, checks if a - package build from cbs is complete and if the build target is saved in the DB. + package build from cbs is complete. """ print(str(msg)) diff --git a/package_updates/mqtt/package_git_release.py b/package_updates/mqtt/package_git_release.py index 4019844..c2eaed1 100644 --- a/package_updates/mqtt/package_git_release.py +++ b/package_updates/mqtt/package_git_release.py @@ -16,7 +16,7 @@ class PackageGitRelease(Package): super().__init__(name, version, release) def get_commit_url(self) -> str: - """Generates a commit rul from commit id. + """Generates a commit url from commit id. Returns: String with git.centos.org commit url. diff --git a/package_updates/utils/helpers.py b/package_updates/utils/helpers.py index 165a939..70996ce 100755 --- a/package_updates/utils/helpers.py +++ b/package_updates/utils/helpers.py @@ -71,7 +71,7 @@ def get_latest_tagged_builds(session: koji.ClientSession, tags:list) -> list: tags: List of koji/cbs tags. Returns: - list of latest package builds. + List of latest package builds. """ builds = [] @@ -88,7 +88,7 @@ def get_koji_tags_info(session: koji.ClientSession, tag_ids: list) -> list: tag_ids: list of koji tag ids. Returns: - list of tuples containing tag id and tag name. + List of tuples containing tag id and tag name. """ tags = [] @@ -192,7 +192,7 @@ def filter_from_tag(tag: str, pkg: str) -> str: def get_latest_version_with_commit(pkg: str, tags: list, branch: str) -> tuple: - """Filters latest git tag of a package. + """Filters latest git tag from a package. Args: pkg: String of the name. @@ -223,7 +223,7 @@ def get_latest_version_with_commit(pkg: str, tags: list, branch: str) -> tuple: def get_git_pkg_version(package: str, branch: str) -> tuple: - """Requests for git tags availables in a repo and searchs + """Requests for git tags available in a repo and search for the latest version tag. Args: @@ -231,7 +231,7 @@ def get_git_pkg_version(package: str, branch: str) -> tuple: branch: String of branch name. Returns: - Tuple that conatins string of pkg version and string of git commit. + Tuple that contains string of pkg version and string of git commit. """ params = { @@ -277,7 +277,7 @@ def get_centos_version(package: str) -> str: def issue_content(package_name: str, distro: str, upstream_nvr: str, current_nvr: str, commit_url: str, cbs_tag: str) -> dict: - """Creates a dictionary for issue content. The dict conatins title, issue_content and tag. + """Creates a dictionary for issue content. The dict contains title, issue_content and tag. Args: issue: Dictionary of issue containing title, issue_content and tag @@ -425,7 +425,7 @@ def comment_on_issue(issue_id: int, comment: dict): def get_issues(tags: list) -> list: - """Requests for all issues that have a speciifc tag. + """Requests for all issues that have a specific tag. Args: tag: List of tags. @@ -486,7 +486,7 @@ def issue_filter(issues: list, package_name: str) -> dict: package_name: String of package name. Returns: - dictionary containing the issue id, package name + Dictionary containing the issue id, package name and version tag, None if there is no package name and package version. """ @@ -511,7 +511,7 @@ def fixed_status_comment(build_url: str, commit_url: str) -> dict: def invalid_status_comment(new_issue_id: int, package: str) -> dict: """ Args: - new_isue_id: Issue id to generate issue link + new_issue_id: Issue id to generate issue link package: String of package nvr """ new_issue_url = urljoin(PAGURE_REPO_URL, f"issue/{new_issue_id}")