From 1b442fed2b0d3235a4b3506b30af344e2cc9a172 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Jul 07 2022 17:05:04 +0000 Subject: [PATCH 1/7] Change project structure & dockerfile base image --- diff --git a/.gitignore b/.gitignore index 3a01f83..9bd9100 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ myenv/ ENV/ env.bak/ venv.bak/ +.vscode/* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 18c3bf4..6d9e9f8 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/centoshyperscale/centos:stream8 +FROM quay.io/centoshyperscale/centos:stream9 RUN dnf update -y && \ dnf install -y \ @@ -16,9 +16,9 @@ WORKDIR /home/app/package-updates RUN mkdir utils RUN chown -R app:wheel $HOME/package-updates -COPY --chown=app:wheel ./package_updates.py $HOME/package-updates +COPY --chown=app:wheel ./package_updates/ $HOME/package-updates/ +RUN ls -la $HOME/package-updates/* COPY --chown=app:wheel ./requirements.txt $HOME/package-updates -COPY --chown=app:wheel utils/* $HOME/package-updates/utils/ RUN pip3 install -r requirements.txt diff --git a/README.md b/README.md index 5ea4e27..4f0d254 100755 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Cert files needed for MQTT, see the Message Broker (MQTT) section: sqlite3.Connection: + """Gets an in-memory sqlite connection. + + Returns: + A sqlite connection. + """ + + return sqlite3.connect("file:mem1?mode=memory&cache=shared", uri=True) + + +def create_package_table(conn: sqlite3.Connection): + """Creates a sql table for hs packages. + + Args: + conn: sqlite3 connection + """ + + cursor = conn.cursor() + sql = """CREATE TABLE IF NOT EXISTS package ( + package_id INTEGER PRIMARY KEY, + name text, + version text, + release text, + build_id INTEGER, + tag_id INTEGER) + """ + + cursor.execute(sql) + conn.commit() + + +def insert_package(package_id: int, name: str, version: str, release: str, + build_id: int, tag_id: int, conn: sqlite3.Connection): + """Inserts a hs package to package table. + + Args: + package_id: CBS package id. + name: String of package name. + version: String of package version. + release: String of package release. + build_id: CBS package build id. + tag_id: CBS package tag id. + conn: sqlite3 connection. + """ + + cursor = conn.cursor() + cursor.execute("""INSERT INTO package( + package_id, + name, + version, + release, + build_id, + tag_id) + VALUES(?, ?, ?, ?, ?, ?)""", + (package_id, name, version, release, build_id, tag_id)) + conn.commit() + + +def select_package(name: str, conn: sqlite3.Connection) -> tuple: + """Gets a hs package from package table. + + Args: + name: String of package name. + conn: sqlite3 connection. + + Returns: + A tuple containing package_id, name, version, release, build_id, tag_id. + """ + + cursor = conn.cursor() + cursor.execute("SELECT * FROM package WHERE name=?", [name]) + + return cursor.fetchone() + + +def select_packages(conn: sqlite3.Connection) -> list: + """Gets all hs packages from database. + + Args: + conn: sqlite3 connection. + + Returns: + A list of tuples of hs packages. + """ + + cursor = conn.cursor() + cursor.execute("SELECT * FROM package") + + return cursor.fetchall() + + +def insert_packages_from_builds(packages: list, conn: sqlite3.Connection): + """Inserts a list of packages in the database. + + Args: + package: list of package builds from cbs. + conn: sqlite3 connection. + """ + + for package in packages: + package_id = package['package_id'] + name = package['package_name'] + version = package['version'] + release = package['release'] + build_id = package['build_id'] + tag_id = package['tag_id'] + + result = select_package(name, conn) + if result: + continue + + insert_package(package_id, name, version, release, build_id, tag_id, conn) + + +def create_tag_table(conn: sqlite3.Connection): + """Creates the tag (from CBS) table. + + Args: + conn: sqlite3 connection. + """ + + cursor = conn.cursor() + sql = """CREATE TABLE IF NOT EXISTS tag ( + tag_id INTEGER PRIMARY KEY, + tag_name TEXT) + """ + + cursor.execute(sql) + conn.commit() + + +def insert_tag(tag_id: int, tag_name: str, conn: sqlite3.Connection): + """Inserts a CBS tag to the tag table. + + Args: + tag_id: CBS hs tag id. + tag_name: String of CBS hs tag name. + conn: sqlite3 connection. + """ + + cursor = conn.cursor() + cursor.execute("""INSERT INTO tag(tag_id, tag_name) VALUES(?, ?)""", (tag_id, tag_name)) + + conn.commit() + + +def insert_tags(tags: list, conn: sqlite3.Connection): + """Inserts a list of tags to the tag table. + + Args: + package: List of cbs tags. + conn: sqlite3 connection. + """ + + for tag in tags: + tag_id = tag[0] + tag_name = tag[1] + + result = select_tag(tag_id, conn) + if result: + continue + + insert_tag(tag_id, tag_name, conn) + + +def select_tag(tag_id: str, conn: sqlite3.Connection) -> tuple: + """Gets a CBS hs tag from tag table. + + Args: + tag_id: CBS hs tag id. + conn: sqlite3 connection. + + Returns: + A tuple containing the tag name. + """ + + cursor = conn.cursor() + cursor.execute("SELECT tag_name FROM tag WHERE tag_id=?", [tag_id]) + + return cursor.fetchone() + + +def create_issue_table(conn: sqlite3.Connection): + """Creates a sql table for pagure issues. + + Args: + conn: sqlite3 connection + """ + + cursor = conn.cursor() + sql = """CREATE TABLE IF NOT EXISTS issue ( + issue_id INTEGER PRIMARY KEY, + package_name text, + version_tag text) + """ + + cursor.execute(sql) + conn.commit() + + +def insert_issue(issue_id: int, package_name: str, version_tag: str, conn: sqlite3.Connection): + """Inserts an issue to issue table. + + Args: + package_name: String of package name. + issue_id: New issue id. + version_tag: String of new centos package version, e.g. dnf-4.7.0-8.el8. + conn: sqlite3 connection. + """ + + cursor = conn.cursor() + cursor.execute("""INSERT INTO issue( + issue_id, + package_name, + version_tag) + VALUES(?, ?, ?)""", + (issue_id, package_name, version_tag)) + conn.commit() + + +def update_issue_row(package_name: str, issue_id: int, version_tag: str, conn: sqlite3.Connection): + """Updates an issue for a specific package in database. + + Args: + package_name: String of package name. + issue_id: New issue id. + version_tag: String of new centos package version, e.g. dnf-4.7.0-8.el8. + conn: sqlite3 connection. + """ + + cursor = conn.cursor() + cursor.execute("UPDATE issue SET issue_id=?, version_tag=? WHERE package_name=?", (issue_id, version_tag, package_name)) + conn.commit() + + +def select_issue(package_name: str, conn: sqlite3.Connection) -> tuple: + """Gets an issue from issue table. + + Args: + package_name: String of package name. + conn: sqlite3 connection. + + Returns: + A tuple containing issue_id, package_name, version_tag. + """ + + cursor = conn.cursor() + cursor.execute("SELECT * FROM issue WHERE package_name=?", [package_name]) + + return cursor.fetchone() \ No newline at end of file diff --git a/package_updates/utils/helpers.py b/package_updates/utils/helpers.py new file mode 100755 index 0000000..ddabf83 --- /dev/null +++ b/package_updates/utils/helpers.py @@ -0,0 +1,407 @@ +import os +from urllib.parse import urljoin +import rpm +import hawkey +import koji +import requests + + +PAGURE_REPO_API_URL = 'https://pagure.io/api/0/centos-sig-hyperscale/package-updates/' +PAGURE_REPO_URL = 'https://pagure.io/centos-sig-hyperscale/package-updates/' +GIT_CENTOS_API_URL = 'https://git.centos.org/api/0/rpms/' +GIT_CENTOS_URL = 'https://git.centos.org/rpms/' + +def get_koji_session(url: str) -> koji.ClientSession: + """Connects to the koji build system. + + Args: + url: String of koji url. + + Returns: + Koji client session. + """ + + return koji.ClientSession(url) + + +def get_tagged_packages(session: koji.ClientSession, tag: int) -> list: + """Requests for a list of packages available in a tag. + + Args: + session: Koji client session. + tag: Tag ID. + + Returns: + List of dictionaries with packages info. + """ + + packages = None + try: + packages = session.listPackages(tagID=tag) + if not packages: + print(f"There is no available packages for {tag}") + return + except koji.GenericError as err: + print(err) + + return packages + + +def join_tagged_packages(session: koji.ClientSession, tags: list) -> list: + """Joins the list of packages avalable in a list of koji tags. + + Args: + session: Koji client session. + tag: List of tag IDs. + + Returns: + List of dictionaries with packages info. + """ + + packages = [] + for tag in tags: + packages += get_tagged_packages(session, tag) + return packages + + +def get_latest_tagged_builds(session: koji.ClientSession, packages: list) -> list: + """Gets the latest cbs build for a list of packages. + + Args: + session: Koji client session. + packages: List of packages. + + Returns: + list of latest package builds. + """ + + builds = [] + for package in packages: + package_name = package['package_name'] + build = session.listTagged(package['tag_id'], latest=True, package=package_name) + if len(build) == 0: + continue + builds.append(build[0]) + + return builds + + +def get_koji_tags(session: koji.ClientSession, tags_id: list) -> list: + """Gets the tag id and tag name from koji cbs. + + Args: + session: Koji client session. + tag_ids: list of koji tag ids. + + Returns: + list of tuples containing tag id and tag name. + """ + + tags = [] + for tag_id in tags_id: + tag = session.getTag(tag_id) + tags.append((tag['id'], tag['name'])) + + return tags + + +def compare_versions(pkg1: str, pkg2: str) -> int: + """Compares package version between two packages. + + Args: + pkg1: String of the first pkg-version-release. E.g. dnf-4.7.0-8.el8 + pkg2: String of the second pkg-version-release. + + Returns: + 1 if pkg1 it's greater than pkg2, 0 if it's equal, -1 if it's lower. + """ + + nevra1 = hawkey.split_nevra(pkg1) + nevra2 = hawkey.split_nevra(pkg2) + + epoch1, v1, r1 = str(nevra1.epoch), nevra1.version, nevra1.release + epoch2, v2, r2 = str(nevra2.epoch), nevra2.version, nevra2.release + + return rpm.labelCompare((epoch1, v1, r1), (epoch2, v2, r2)) + + +def split_package(pkg: str): + """Splits a package into name, version and release. + + Args: + pkg: String of package. E.g. dnf-4.7.0-8.el8 + + Returns: + Tuple containing package name, version and release. + """ + + subj = hawkey.Subject(pkg) + nevra_possibilities = subj.get_nevra_possibilities() + epel = 'el8' + + for nevra in nevra_possibilities: + if epel in str(nevra.release): + return (str(nevra.name), str(nevra.version), str(nevra.release)) + + return None + + +def filter_from_tag(tag: str, pkg: str) -> str: + """Filters the package name-version-release from a centos release tag. + + Args: + tag: String of the git tag. + pkg: String of the name of the package to filter. + + Retruns: + String of the package name-version-release. + """ + + index = tag.find(pkg) + return tag[index:] + + +def get_latest_version_with_commit(pkg: str, tags: list, branch: str) -> tuple: + """Filters latest git tag of a package. + + Args: + tags: List of git tags. + branch: String of git branch. + + Returns: + Tuple of latest version in a tag (name-version-release, commit). + """ + + branch_tags = {key: val for key, val in tags.items() if branch in key} + tags = list(branch_tags.items()) #list of tuples: [(tag, commit)] + + if len(tags) == 0: + return (None, None) + + newer = filter_from_tag(tags[0][0], pkg) + latest_commit = tags[0][1] + + for i in range(1, len(tags)): + curr = filter_from_tag(tags[i][0], pkg) + result = compare_versions(curr, newer) + if result == 1: + newer = curr + latest_commit = tags[i][1] + + return (newer, latest_commit) + + +def get_git_pkg_version(package: str, branch: str) -> tuple: + """Requests for git tags availables in a repo and searchs + for the latest version tag. + + Args: + package: String of package name. + branch: String of branch name. + + Returns: + Tuple that conatins string of pkg version and string of git commit. + """ + + params = { + 'with_commits': 'true' + } + url = urljoin(GIT_CENTOS_API_URL, f'{package}/git/tags') + + try: + res = requests.get(url=url, params=params) + res.raise_for_status() + except requests.exceptions.RequestException as err: + print(err) + return (None, None) + + tags = res.json()['tags'] + latest_version, commit = get_latest_version_with_commit(package, tags, branch) + + if latest_version == None: + return (None, None) + + return (latest_version, commit) + + +def create_ticket(pkg: str, upstream_version: str, current_version: str, cbs_tag_name: str, commit: str): + """Creates a ticket on paguire.io repo. + + Args: + pkg: String of package name. + tag_version: String of package version. + """ + + commit_url = urljoin(GIT_CENTOS_URL, f'{pkg}/tree/{commit}') + issue = { + 'title': f'{upstream_version} is available', + 'issue_content': f"""Latest upstream release: {upstream_version} + Current version/release: {current_version} + URL: {commit_url}""", + 'tag': f'{cbs_tag_name},{upstream_version},{pkg}' + } + url = urljoin(PAGURE_REPO_API_URL, 'new_issue') + token = os.environ['PAGURE_API_KEY'] + headers = {'Authorization': f'access_token {token}'} + print('Creating ticket on pagure.io...') + + try: + res = requests.post(url=url, data=issue, headers=headers) + res.raise_for_status() + except requests.exceptions.HTTPError as errh: + if res.status_code == 401: + print(errh) + raise SystemExit(f'{errh}\nMake sure you have the correct API token') + print(errh) + return + except requests.exceptions.RequestException as err: + print(err) + return + + print('ticket created') + return res.json()['issue'] + + +def is_issue_Open(tag: str) -> bool: + """Verify if the issue for a specific tag is open. + + Args: + tag: String of issue tag. + + Returns: + True if the issue for that package update + has been created, false otherwise. + """ + + params = { + 'status': 'Open', + 'tags': tag + } + url = urljoin(PAGURE_REPO_API_URL, 'issues') + + try: + res = requests.get(url=url, params=params) + res.raise_for_status() + except requests.exceptions.RequestException as err: + print(err) + return True + + total_issues = res.json()['total_issues'] + if total_issues > 0: + return True + + return False + + +def comment_on_issue(prev_issue_id: int, new_issue_id: int, version_tag: str): + """Adds a comment to a specific pagure.io issue. + + Args: + prev_issue_id: Issue id of the issue to add a comment, + new_issue_id: Issue Id of the new created issue, lo leave a link of the new issue. + version_tag: String of new package version. + """ + + new_issue_url = urljoin(PAGURE_REPO_URL, f'issue/{new_issue_id}') + comment = { + 'comment': f"""{version_tag} is available and this issue still open. + New issue has been created for the newer version. + URL: {new_issue_url}""" + } + url = urljoin(PAGURE_REPO_API_URL, f'issue/{prev_issue_id}/comment') + token = os.environ['PAGURE_API_KEY'] + headers = {'Authorization': f'access_token {token}'} + + try: + res = requests.post(url=url, data=comment, headers=headers) + res.raise_for_status() + except requests.exceptions.HTTPError as errh: + if res.status_code == 401: + print(errh) + raise SystemExit(f'{errh}\nMake sure you have the correct API token') + print(errh) + return + except requests.exceptions.RequestException as err: + print(err) + return + + print(f'Comment added on issue {prev_issue_id}') + + +def get_issues(tags: list) -> list: + """Requests for all issues that have a speciifc tag. + + Args: + tag: List of tags. + + Returns: + List of pagure.io issues. + """ + + params = { + 'status': 'Open', + 'tags': tags + } + url = urljoin(PAGURE_REPO_API_URL, 'issues') + + try: + res = requests.get(url=url, params=params) + res.raise_for_status() + except requests.exceptions.RequestException as err: + print(err) + return None + + issues = res.json()['issues'] + return issues + + +def close_issue(issue_id: int): + """Closes a pagure.io issue. + + Args: + issue_id: Issue id. + """ + + data = { + 'status': 'Closed', + 'close_status': 'Invalid' + } + url = urljoin(PAGURE_REPO_API_URL, f'issue/{issue_id}/status') + token = os.environ['PAGURE_API_KEY'] + headers = {'Authorization': f'access_token {token}'} + + try: + res = requests.post(url=url, data=data, headers=headers) + res.raise_for_status() + except requests.exceptions.RequestException as err: + print(err) + print(res.json()) + return + print('issue closed') + + +def issue_filter(package_name: str, issues: list) -> dict: + """Filters and returns an issue if has a tag with a + package name and a package version. + + Args: + package_name: String of package name. + issues: List of pagure.io issues. + + Returns: + dictionary containing the issue id, package name + and version tag, None if there is no package name + and package version. + """ + + version_tag = '' + for issue in issues: + c = 0 + for tag in issue['tags']: + if package_name in tag: + c += 1 + if tag != package_name: + version_tag = tag + if c == 2: + return {'issue_id': issue['id'], 'package_name': issue, 'version_tag': version_tag} + return None \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 883e75c..eb2feb9 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ koji>=1.27.1 requests>=2.27.1 paho-mqtt>=1.6.1 +fedora-messaging>=3.0.2 \ No newline at end of file diff --git a/test_mqtt_pub/publish.py b/test_mqtt_pub/publish.py deleted file mode 100755 index 0ec4e46..0000000 --- a/test_mqtt_pub/publish.py +++ /dev/null @@ -1,60 +0,0 @@ -import time -import json -import random -from paho.mqtt import client as mqtt - - -broker = 'localhost' -port = 1883 -topic = "git.centos.org/git.tag.creation" -client_id = f'client-py-{random.randint(0, 1000)}' - - -def on_connect(client, user_data, flags, rc): - if rc == 0: - print(f'Connected to MQTT Broker') - else: - print(f'Failed to connect, return code {rc}') - - -def connect_mqtt(): - client = mqtt.Client(client_id) - client.on_connect = on_connect - client.connect(broker, port) - return client - - -def publish(client): - msg_count = 1 - msg = b'' - while True: - time.sleep(5) - if msg_count == 1: - msg = b'{"repo": {"custom_keys": [], "name": "selinux-policy", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/selinux-policy", "url_path": "rpms/selinux-policy", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/selinux-policy-3.14.3-93.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' - elif msg_count == 2: - msg = b'{"repo": {"custom_keys": [], "name": "dnf", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/dnf", "url_path": "rpms/dnf", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/dnf-4.7.0-9.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' - else: - msg = b'{"repo": {"custom_keys": [], "name": "dnf", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/dnf", "url_path": "rpms/dnf", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/dnf-4.7.0-1.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' - - result = client.publish(topic, msg) - status = result[0] - if status == 0: - print(f"Sent `{msg}` to topic `{topic}`") - else: - print(f"Failed to sent message to topic {topic}") - - if msg_count == 3: - msg_count = 0 - - msg_count += 1 - print('----------------------------------------------') - - -def run(): - client = connect_mqtt() - client.loop_start() - publish(client) - - -if __name__ == '__main__': - run() diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100755 index 0000000..33833b1 --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,98 @@ +import unittest +from package_updates.utils.helpers import ( + get_latest_version_with_commit, + compare_versions, + filter_from_tag +) + + +class TestMain(unittest.TestCase): + + def test_compare_versions(self): + greater_list = ['dnf-4.7.0-5.el8', 'mesa-20.3.3-2.1.hs.el8'] + lower_list = ['dnf-4.7.0-4.1.hsx.el8', 'mesa-19.3.4-2.el8'] + + for greater, lower in zip(greater_list, lower_list): + self.assertEqual(compare_versions(greater, lower), 1) + + def test_filter_from_tag(self): + tags = { + 'dnf': 'imports/c8s/dnf-4.7.0-7.el8', + 'clang': 'imports/c8s-stream-rhel8/clang-13.0.1-1.module+el8.6.0+14118+d530a951', + 'dracut': 'imports/c8s/dracut-049-201.git20220131.el8', + 'libuser': 'libuser-0.62-23.2.hs.el8', + 'util-linux': 'imports/c8s/util-linux-2.32.1-34.el8', + 'createrepo_c': 'imports/c8s/createrepo_c-0.17.7-4.el8', + 'rpm': 'imports/c8s/rpm-4.14.el8' + } + + results = [ + 'dnf-4.7.0-7.el8', + 'clang-13.0.1-1.module+el8.6.0+14118+d530a951', + 'dracut-049-201.git20220131.el8', + 'libuser-0.62-23.2.hs.el8', + 'util-linux-2.32.1-34.el8', + 'createrepo_c-0.17.7-4.el8', + 'rpm-4.14.el8' + ] + + for tag, result in zip(tags, results): + self.assertEqual(filter_from_tag(tags[tag], tag), result) + + def test_get_latest_git_tag(self): + tags = { + "imports/c8/rpm-4.14.3-13.el8": "b445f2ffef77c56fb45f6e679d1894d22867d501", + "imports/c8/rpm-4.14.3-14.el8_4": "41043c8511245c3abba654656d2161a2ad68791a", + "imports/c8/rpm-4.14.3-19.el8": "00810bfb118747fbe980b81845009dbfcfcc8450", + "imports/c8/rpm-4.14.3-19.el8_5.2": "377311e10f5d8b70af6d6d59a8d2e2e532893492", + "imports/c8/rpm-4.14.3-4.el8": "b7b8f7e8f5cdbef3992c8ee853ecc444df930503", + "imports/c8s/rpm-4.14.3-14.el8_4": "69c9a1638657acea0c70decddb0b76959032bbea", + "imports/c8s/rpm-4.14.3-15.el8": "5a7695fcfeb883271f76b96178c0fc9de9c3697d", + "imports/c8s/rpm-4.14.3-17.el8": "bfc6f7541d6e6229df6d7951d063c7631850cdcb", + "imports/c8s/rpm-4.14.3-18.el8": "d8c505ff117b10aa0905dd8567c031610a00c928", + "imports/c8s/rpm-4.14.3-19.el8": "dcdfdde122434aaba3986a8ed22056b072c9b9cd", + "imports/c8s/rpm-4.14.3-20.el8": "c7b760fdfae67300dc7d06ba3ad7fce7ad9a6c20", + "imports/c8s/rpm-4.14.3-21.el8": "a9e8d80332d37e00d3e07237cd042dcf248991de", + "imports/c8s/rpm-4.14.3-22.el8": "63d1365d625c4a6bcf3eff7b5488f5554ad4ca44", + "imports/c8s/rpm-4.14.3-3.el8": "35cbef94d0e7d64f3dbec7c0e4d533ebefd5f780", + "imports/c8s/rpm-4.14.3-4.el8": "09d0f647d30af7497fa2e9d45b1254280071fc7a", + "imports/c9-beta/rpm-4.16.1.3-11.el9": "137a597a82550d8c9563615e057ee6679e497ce6", + "imports/c9-beta/rpm-4.16.1.3-7.el9": "813822951aaa7e26eec176925250d9ce7430d62b", + "imports/c9-beta/rpm-4.16.1.3-9.el9": "3f3ddd0c8927e60ed546707bac1303a44035f0ec" + } + + tags2 = { + "imports/c8/rpm-4.14.3-13.el8": "b445f2ffef77c56fb45f6e679d1894d22867d501", + "imports/c8/rpm-4.14.3-14.el8_4": "41043c8511245c3abba654656d2161a2ad68791a", + "imports/c8/rpm-4.14.3-19.el8": "00810bfb118747fbe980b81845009dbfcfcc8450", + "imports/c8/rpm-4.14.3-19.el8_5.2": "377311e10f5d8b70af6d6d59a8d2e2e532893492", + "imports/c8/rpm-4.14.3-4.el8": "b7b8f7e8f5cdbef3992c8ee853ecc444df930503", + "imports/c8s/rpm-4.14.3-14.el8_4": "69c9a1638657acea0c70decddb0b76959032bbea", + "imports/c8s/rpm-4.14.3-15.el8": "5a7695fcfeb883271f76b96178c0fc9de9c3697d", + "imports/c8s/rpm-4.14.3-17.el8": "bfc6f7541d6e6229df6d7951d063c7631850cdcb", + "imports/c8s/rpm-4.14.3-18.el8": "d8c505ff117b10aa0905dd8567c031610a00c928", + "imports/c8s/rpm-4.14.3-19.el8": "dcdfdde122434aaba3986a8ed22056b072c9b9cd", + "imports/c8s/rpm-4.14.3-20.el8": "c7b760fdfae67300dc7d06ba3ad7fce7ad9a6c20", + "imports/c8s/rpm-4.14.3-21.el8": "a9e8d80332d37e00d3e07237cd042dcf248991de", + "imports/c8s/rpm-4.14.3-22.el8": "63d1365d625c4a6bcf3eff7b5488f5554ad4ca44", + "imports/c9-beta/rpm-4.16.1.3-11.el9": "137a597a82550d8c9563615e057ee6679e497ce6", + "imports/c9-beta/rpm-4.16.1.3-7.el9": "813822951aaa7e26eec176925250d9ce7430d62b", + "imports/c9-beta/rpm-4.16.1.3-9.el9": "3f3ddd0c8927e60ed546707bac1303a44035f0ec" + } + + tags3 = { + "imports/c8s/rpm-4.14.3-21.el8": "a9e8d80332d37e00d3e07237cd042dcf248991de", + "imports/c8s/rpm-4.14.3-22.el8": "63d1365d625c4a6bcf3eff7b5488f5554ad4ca44" + } + + tags4 = { + "imports/c8s/rpm-4.14.3-21.el8": "a9e8d80332d37e00d3e07237cd042dcf248991de" + } + + self.assertEqual(get_latest_version_with_commit('rpm', tags, 'c8s'), ('rpm-4.14.3-22.el8', '63d1365d625c4a6bcf3eff7b5488f5554ad4ca44')) + self.assertEqual(get_latest_version_with_commit('rpm', tags2, 'c8s'), ('rpm-4.14.3-22.el8', '63d1365d625c4a6bcf3eff7b5488f5554ad4ca44')) + self.assertEqual(get_latest_version_with_commit('rpm', tags3, 'c8s'), ('rpm-4.14.3-22.el8', '63d1365d625c4a6bcf3eff7b5488f5554ad4ca44')) + self.assertEqual(get_latest_version_with_commit('rpm', tags4, 'c8s'), ('rpm-4.14.3-21.el8', 'a9e8d80332d37e00d3e07237cd042dcf248991de')) + + + diff --git a/tests/test_mqtt_pub/publish.py b/tests/test_mqtt_pub/publish.py new file mode 100755 index 0000000..0ec4e46 --- /dev/null +++ b/tests/test_mqtt_pub/publish.py @@ -0,0 +1,60 @@ +import time +import json +import random +from paho.mqtt import client as mqtt + + +broker = 'localhost' +port = 1883 +topic = "git.centos.org/git.tag.creation" +client_id = f'client-py-{random.randint(0, 1000)}' + + +def on_connect(client, user_data, flags, rc): + if rc == 0: + print(f'Connected to MQTT Broker') + else: + print(f'Failed to connect, return code {rc}') + + +def connect_mqtt(): + client = mqtt.Client(client_id) + client.on_connect = on_connect + client.connect(broker, port) + return client + + +def publish(client): + msg_count = 1 + msg = b'' + while True: + time.sleep(5) + if msg_count == 1: + msg = b'{"repo": {"custom_keys": [], "name": "selinux-policy", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/selinux-policy", "url_path": "rpms/selinux-policy", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/selinux-policy-3.14.3-93.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' + elif msg_count == 2: + msg = b'{"repo": {"custom_keys": [], "name": "dnf", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/dnf", "url_path": "rpms/dnf", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/dnf-4.7.0-9.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' + else: + msg = b'{"repo": {"custom_keys": [], "name": "dnf", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/dnf", "url_path": "rpms/dnf", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/dnf-4.7.0-1.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' + + result = client.publish(topic, msg) + status = result[0] + if status == 0: + print(f"Sent `{msg}` to topic `{topic}`") + else: + print(f"Failed to sent message to topic {topic}") + + if msg_count == 3: + msg_count = 0 + + msg_count += 1 + print('----------------------------------------------') + + +def run(): + client = connect_mqtt() + client.loop_start() + publish(client) + + +if __name__ == '__main__': + run() diff --git a/utils/__init__.py b/utils/__init__.py deleted file mode 100755 index e69de29..0000000 --- a/utils/__init__.py +++ /dev/null diff --git a/utils/db.py b/utils/db.py deleted file mode 100755 index 695bf8c..0000000 --- a/utils/db.py +++ /dev/null @@ -1,253 +0,0 @@ -import sqlite3 - - -def get_connection() -> sqlite3.Connection: - """Gets an in-memory sqlite connection. - - Returns: - A sqlite connection. - """ - - return sqlite3.connect("file:mem1?mode=memory&cache=shared", uri=True) - - -def create_package_table(conn: sqlite3.Connection): - """Creates a sql table for hs packages. - - Args: - conn: sqlite3 connection - """ - - cursor = conn.cursor() - sql = """CREATE TABLE IF NOT EXISTS package ( - package_id INTEGER PRIMARY KEY, - name text, - version text, - release text, - build_id INTEGER, - tag_id INTEGER) - """ - - cursor.execute(sql) - conn.commit() - - -def insert_package(package_id: int, name: str, version: str, release: str, - build_id: int, tag_id: int, conn: sqlite3.Connection): - """Inserts a hs package to package table. - - Args: - package_id: CBS package id. - name: String of package name. - version: String of package version. - release: String of package release. - build_id: CBS package build id. - tag_id: CBS package tag id. - conn: sqlite3 connection. - """ - - cursor = conn.cursor() - cursor.execute("""INSERT INTO package( - package_id, - name, - version, - release, - build_id, - tag_id) - VALUES(?, ?, ?, ?, ?, ?)""", - (package_id, name, version, release, build_id, tag_id)) - conn.commit() - - -def select_package(name: str, conn: sqlite3.Connection) -> tuple: - """Gets a hs package from package table. - - Args: - name: String of package name. - conn: sqlite3 connection. - - Returns: - A tuple containing package_id, name, version, release, build_id, tag_id. - """ - - cursor = conn.cursor() - cursor.execute("SELECT * FROM package WHERE name=?", [name]) - - return cursor.fetchone() - - -def select_packages(conn: sqlite3.Connection) -> list: - """Gets all hs packages from database. - - Args: - conn: sqlite3 connection. - - Returns: - A list of tuples of hs packages. - """ - - cursor = conn.cursor() - cursor.execute("SELECT * FROM package") - - return cursor.fetchall() - - -def insert_packages_from_builds(packages: list, conn: sqlite3.Connection): - """Inserts a list of packages in the database. - - Args: - package: list of package builds from cbs. - conn: sqlite3 connection. - """ - - for package in packages: - package_id = package['package_id'] - name = package['package_name'] - version = package['version'] - release = package['release'] - build_id = package['build_id'] - tag_id = package['tag_id'] - - result = select_package(name, conn) - if result: - continue - - insert_package(package_id, name, version, release, build_id, tag_id, conn) - - -def create_tag_table(conn: sqlite3.Connection): - """Creates the tag (from CBS) table. - - Args: - conn: sqlite3 connection. - """ - - cursor = conn.cursor() - sql = """CREATE TABLE IF NOT EXISTS tag ( - tag_id INTEGER PRIMARY KEY, - tag_name TEXT) - """ - - cursor.execute(sql) - conn.commit() - - -def insert_tag(tag_id: int, tag_name: str, conn: sqlite3.Connection): - """Inserts a CBS tag to the tag table. - - Args: - tag_id: CBS hs tag id. - tag_name: String of CBS hs tag name. - conn: sqlite3 connection. - """ - - cursor = conn.cursor() - cursor.execute("""INSERT INTO tag(tag_id, tag_name) VALUES(?, ?)""", (tag_id, tag_name)) - - conn.commit() - - -def insert_tags(tags: list, conn: sqlite3.Connection): - """Inserts a list of tags to the tag table. - - Args: - package: List of cbs tags. - conn: sqlite3 connection. - """ - - for tag in tags: - tag_id = tag[0] - tag_name = tag[1] - - result = select_tag(tag_id, conn) - if result: - continue - - insert_tag(tag_id, tag_name, conn) - - -def select_tag(tag_id: str, conn: sqlite3.Connection) -> tuple: - """Gets a CBS hs tag from tag table. - - Args: - tag_id: CBS hs tag id. - conn: sqlite3 connection. - - Returns: - A tuple containing the tag name. - """ - - cursor = conn.cursor() - cursor.execute("SELECT tag_name FROM tag WHERE tag_id=?", [tag_id]) - - return cursor.fetchone() - - -def create_issue_table(conn: sqlite3.Connection): - """Creates a sql table for pagure issues. - - Args: - conn: sqlite3 connection - """ - - cursor = conn.cursor() - sql = """CREATE TABLE IF NOT EXISTS issue ( - issue_id INTEGER PRIMARY KEY, - package_name text, - version_tag text) - """ - - cursor.execute(sql) - conn.commit() - - -def insert_issue(issue_id: int, package_name: str, version_tag: str, conn: sqlite3.Connection): - """Inserts an issue to issue table. - - Args: - package_name: String of package name. - issue_id: New issue id. - version_tag: String of new centos package version, e.g. dnf-4.7.0-8.el8. - conn: sqlite3 connection. - """ - - cursor = conn.cursor() - cursor.execute("""INSERT INTO issue( - issue_id, - package_name, - version_tag) - VALUES(?, ?, ?)""", - (issue_id, package_name, version_tag)) - conn.commit() - - -def update_issue_row(package_name: str, issue_id: int, version_tag: str, conn: sqlite3.Connection): - """Updates an issue for a specific package in database. - - Args: - package_name: String of package name. - issue_id: New issue id. - version_tag: String of new centos package version, e.g. dnf-4.7.0-8.el8. - conn: sqlite3 connection. - """ - - cursor = conn.cursor() - cursor.execute("UPDATE issue SET issue_id=?, version_tag=? WHERE package_name=?", (issue_id, version_tag, package_name)) - conn.commit() - - -def select_issue(package_name: str, conn: sqlite3.Connection) -> tuple: - """Gets an issue from issue table. - - Args: - package_name: String of package name. - conn: sqlite3 connection. - - Returns: - A tuple containing issue_id, package_name, version_tag. - """ - - cursor = conn.cursor() - cursor.execute("SELECT * FROM issue WHERE package_name=?", [package_name]) - - return cursor.fetchone() \ No newline at end of file diff --git a/utils/helpers.py b/utils/helpers.py deleted file mode 100755 index ddabf83..0000000 --- a/utils/helpers.py +++ /dev/null @@ -1,407 +0,0 @@ -import os -from urllib.parse import urljoin -import rpm -import hawkey -import koji -import requests - - -PAGURE_REPO_API_URL = 'https://pagure.io/api/0/centos-sig-hyperscale/package-updates/' -PAGURE_REPO_URL = 'https://pagure.io/centos-sig-hyperscale/package-updates/' -GIT_CENTOS_API_URL = 'https://git.centos.org/api/0/rpms/' -GIT_CENTOS_URL = 'https://git.centos.org/rpms/' - -def get_koji_session(url: str) -> koji.ClientSession: - """Connects to the koji build system. - - Args: - url: String of koji url. - - Returns: - Koji client session. - """ - - return koji.ClientSession(url) - - -def get_tagged_packages(session: koji.ClientSession, tag: int) -> list: - """Requests for a list of packages available in a tag. - - Args: - session: Koji client session. - tag: Tag ID. - - Returns: - List of dictionaries with packages info. - """ - - packages = None - try: - packages = session.listPackages(tagID=tag) - if not packages: - print(f"There is no available packages for {tag}") - return - except koji.GenericError as err: - print(err) - - return packages - - -def join_tagged_packages(session: koji.ClientSession, tags: list) -> list: - """Joins the list of packages avalable in a list of koji tags. - - Args: - session: Koji client session. - tag: List of tag IDs. - - Returns: - List of dictionaries with packages info. - """ - - packages = [] - for tag in tags: - packages += get_tagged_packages(session, tag) - return packages - - -def get_latest_tagged_builds(session: koji.ClientSession, packages: list) -> list: - """Gets the latest cbs build for a list of packages. - - Args: - session: Koji client session. - packages: List of packages. - - Returns: - list of latest package builds. - """ - - builds = [] - for package in packages: - package_name = package['package_name'] - build = session.listTagged(package['tag_id'], latest=True, package=package_name) - if len(build) == 0: - continue - builds.append(build[0]) - - return builds - - -def get_koji_tags(session: koji.ClientSession, tags_id: list) -> list: - """Gets the tag id and tag name from koji cbs. - - Args: - session: Koji client session. - tag_ids: list of koji tag ids. - - Returns: - list of tuples containing tag id and tag name. - """ - - tags = [] - for tag_id in tags_id: - tag = session.getTag(tag_id) - tags.append((tag['id'], tag['name'])) - - return tags - - -def compare_versions(pkg1: str, pkg2: str) -> int: - """Compares package version between two packages. - - Args: - pkg1: String of the first pkg-version-release. E.g. dnf-4.7.0-8.el8 - pkg2: String of the second pkg-version-release. - - Returns: - 1 if pkg1 it's greater than pkg2, 0 if it's equal, -1 if it's lower. - """ - - nevra1 = hawkey.split_nevra(pkg1) - nevra2 = hawkey.split_nevra(pkg2) - - epoch1, v1, r1 = str(nevra1.epoch), nevra1.version, nevra1.release - epoch2, v2, r2 = str(nevra2.epoch), nevra2.version, nevra2.release - - return rpm.labelCompare((epoch1, v1, r1), (epoch2, v2, r2)) - - -def split_package(pkg: str): - """Splits a package into name, version and release. - - Args: - pkg: String of package. E.g. dnf-4.7.0-8.el8 - - Returns: - Tuple containing package name, version and release. - """ - - subj = hawkey.Subject(pkg) - nevra_possibilities = subj.get_nevra_possibilities() - epel = 'el8' - - for nevra in nevra_possibilities: - if epel in str(nevra.release): - return (str(nevra.name), str(nevra.version), str(nevra.release)) - - return None - - -def filter_from_tag(tag: str, pkg: str) -> str: - """Filters the package name-version-release from a centos release tag. - - Args: - tag: String of the git tag. - pkg: String of the name of the package to filter. - - Retruns: - String of the package name-version-release. - """ - - index = tag.find(pkg) - return tag[index:] - - -def get_latest_version_with_commit(pkg: str, tags: list, branch: str) -> tuple: - """Filters latest git tag of a package. - - Args: - tags: List of git tags. - branch: String of git branch. - - Returns: - Tuple of latest version in a tag (name-version-release, commit). - """ - - branch_tags = {key: val for key, val in tags.items() if branch in key} - tags = list(branch_tags.items()) #list of tuples: [(tag, commit)] - - if len(tags) == 0: - return (None, None) - - newer = filter_from_tag(tags[0][0], pkg) - latest_commit = tags[0][1] - - for i in range(1, len(tags)): - curr = filter_from_tag(tags[i][0], pkg) - result = compare_versions(curr, newer) - if result == 1: - newer = curr - latest_commit = tags[i][1] - - return (newer, latest_commit) - - -def get_git_pkg_version(package: str, branch: str) -> tuple: - """Requests for git tags availables in a repo and searchs - for the latest version tag. - - Args: - package: String of package name. - branch: String of branch name. - - Returns: - Tuple that conatins string of pkg version and string of git commit. - """ - - params = { - 'with_commits': 'true' - } - url = urljoin(GIT_CENTOS_API_URL, f'{package}/git/tags') - - try: - res = requests.get(url=url, params=params) - res.raise_for_status() - except requests.exceptions.RequestException as err: - print(err) - return (None, None) - - tags = res.json()['tags'] - latest_version, commit = get_latest_version_with_commit(package, tags, branch) - - if latest_version == None: - return (None, None) - - return (latest_version, commit) - - -def create_ticket(pkg: str, upstream_version: str, current_version: str, cbs_tag_name: str, commit: str): - """Creates a ticket on paguire.io repo. - - Args: - pkg: String of package name. - tag_version: String of package version. - """ - - commit_url = urljoin(GIT_CENTOS_URL, f'{pkg}/tree/{commit}') - issue = { - 'title': f'{upstream_version} is available', - 'issue_content': f"""Latest upstream release: {upstream_version} - Current version/release: {current_version} - URL: {commit_url}""", - 'tag': f'{cbs_tag_name},{upstream_version},{pkg}' - } - url = urljoin(PAGURE_REPO_API_URL, 'new_issue') - token = os.environ['PAGURE_API_KEY'] - headers = {'Authorization': f'access_token {token}'} - print('Creating ticket on pagure.io...') - - try: - res = requests.post(url=url, data=issue, headers=headers) - res.raise_for_status() - except requests.exceptions.HTTPError as errh: - if res.status_code == 401: - print(errh) - raise SystemExit(f'{errh}\nMake sure you have the correct API token') - print(errh) - return - except requests.exceptions.RequestException as err: - print(err) - return - - print('ticket created') - return res.json()['issue'] - - -def is_issue_Open(tag: str) -> bool: - """Verify if the issue for a specific tag is open. - - Args: - tag: String of issue tag. - - Returns: - True if the issue for that package update - has been created, false otherwise. - """ - - params = { - 'status': 'Open', - 'tags': tag - } - url = urljoin(PAGURE_REPO_API_URL, 'issues') - - try: - res = requests.get(url=url, params=params) - res.raise_for_status() - except requests.exceptions.RequestException as err: - print(err) - return True - - total_issues = res.json()['total_issues'] - if total_issues > 0: - return True - - return False - - -def comment_on_issue(prev_issue_id: int, new_issue_id: int, version_tag: str): - """Adds a comment to a specific pagure.io issue. - - Args: - prev_issue_id: Issue id of the issue to add a comment, - new_issue_id: Issue Id of the new created issue, lo leave a link of the new issue. - version_tag: String of new package version. - """ - - new_issue_url = urljoin(PAGURE_REPO_URL, f'issue/{new_issue_id}') - comment = { - 'comment': f"""{version_tag} is available and this issue still open. - New issue has been created for the newer version. - URL: {new_issue_url}""" - } - url = urljoin(PAGURE_REPO_API_URL, f'issue/{prev_issue_id}/comment') - token = os.environ['PAGURE_API_KEY'] - headers = {'Authorization': f'access_token {token}'} - - try: - res = requests.post(url=url, data=comment, headers=headers) - res.raise_for_status() - except requests.exceptions.HTTPError as errh: - if res.status_code == 401: - print(errh) - raise SystemExit(f'{errh}\nMake sure you have the correct API token') - print(errh) - return - except requests.exceptions.RequestException as err: - print(err) - return - - print(f'Comment added on issue {prev_issue_id}') - - -def get_issues(tags: list) -> list: - """Requests for all issues that have a speciifc tag. - - Args: - tag: List of tags. - - Returns: - List of pagure.io issues. - """ - - params = { - 'status': 'Open', - 'tags': tags - } - url = urljoin(PAGURE_REPO_API_URL, 'issues') - - try: - res = requests.get(url=url, params=params) - res.raise_for_status() - except requests.exceptions.RequestException as err: - print(err) - return None - - issues = res.json()['issues'] - return issues - - -def close_issue(issue_id: int): - """Closes a pagure.io issue. - - Args: - issue_id: Issue id. - """ - - data = { - 'status': 'Closed', - 'close_status': 'Invalid' - } - url = urljoin(PAGURE_REPO_API_URL, f'issue/{issue_id}/status') - token = os.environ['PAGURE_API_KEY'] - headers = {'Authorization': f'access_token {token}'} - - try: - res = requests.post(url=url, data=data, headers=headers) - res.raise_for_status() - except requests.exceptions.RequestException as err: - print(err) - print(res.json()) - return - print('issue closed') - - -def issue_filter(package_name: str, issues: list) -> dict: - """Filters and returns an issue if has a tag with a - package name and a package version. - - Args: - package_name: String of package name. - issues: List of pagure.io issues. - - Returns: - dictionary containing the issue id, package name - and version tag, None if there is no package name - and package version. - """ - - version_tag = '' - for issue in issues: - c = 0 - for tag in issue['tags']: - if package_name in tag: - c += 1 - if tag != package_name: - version_tag = tag - if c == 2: - return {'issue_id': issue['id'], 'package_name': issue, 'version_tag': version_tag} - return None \ No newline at end of file diff --git a/utils/test_helpers.py b/utils/test_helpers.py deleted file mode 100755 index b5d3dc9..0000000 --- a/utils/test_helpers.py +++ /dev/null @@ -1,98 +0,0 @@ -import unittest -from helpers import ( - get_latest_version_with_commit, - compare_versions, - filter_from_tag -) - - -class TestMain(unittest.TestCase): - - def test_compare_versions(self): - greater_list = ['dnf-4.7.0-5.el8', 'mesa-20.3.3-2.1.hs.el8'] - lower_list = ['dnf-4.7.0-4.1.hsx.el8', 'mesa-19.3.4-2.el8'] - - for greater, lower in zip(greater_list, lower_list): - self.assertEqual(compare_versions(greater, lower), 1) - - def test_filter_from_tag(self): - tags = { - 'dnf': 'imports/c8s/dnf-4.7.0-7.el8', - 'clang': 'imports/c8s-stream-rhel8/clang-13.0.1-1.module+el8.6.0+14118+d530a951', - 'dracut': 'imports/c8s/dracut-049-201.git20220131.el8', - 'libuser': 'libuser-0.62-23.2.hs.el8', - 'util-linux': 'imports/c8s/util-linux-2.32.1-34.el8', - 'createrepo_c': 'imports/c8s/createrepo_c-0.17.7-4.el8', - 'rpm': 'imports/c8s/rpm-4.14.el8' - } - - results = [ - 'dnf-4.7.0-7.el8', - 'clang-13.0.1-1.module+el8.6.0+14118+d530a951', - 'dracut-049-201.git20220131.el8', - 'libuser-0.62-23.2.hs.el8', - 'util-linux-2.32.1-34.el8', - 'createrepo_c-0.17.7-4.el8', - 'rpm-4.14.el8' - ] - - for tag, result in zip(tags, results): - self.assertEqual(filter_from_tag(tags[tag], tag), result) - - def test_get_latest_git_tag(self): - tags = { - "imports/c8/rpm-4.14.3-13.el8": "b445f2ffef77c56fb45f6e679d1894d22867d501", - "imports/c8/rpm-4.14.3-14.el8_4": "41043c8511245c3abba654656d2161a2ad68791a", - "imports/c8/rpm-4.14.3-19.el8": "00810bfb118747fbe980b81845009dbfcfcc8450", - "imports/c8/rpm-4.14.3-19.el8_5.2": "377311e10f5d8b70af6d6d59a8d2e2e532893492", - "imports/c8/rpm-4.14.3-4.el8": "b7b8f7e8f5cdbef3992c8ee853ecc444df930503", - "imports/c8s/rpm-4.14.3-14.el8_4": "69c9a1638657acea0c70decddb0b76959032bbea", - "imports/c8s/rpm-4.14.3-15.el8": "5a7695fcfeb883271f76b96178c0fc9de9c3697d", - "imports/c8s/rpm-4.14.3-17.el8": "bfc6f7541d6e6229df6d7951d063c7631850cdcb", - "imports/c8s/rpm-4.14.3-18.el8": "d8c505ff117b10aa0905dd8567c031610a00c928", - "imports/c8s/rpm-4.14.3-19.el8": "dcdfdde122434aaba3986a8ed22056b072c9b9cd", - "imports/c8s/rpm-4.14.3-20.el8": "c7b760fdfae67300dc7d06ba3ad7fce7ad9a6c20", - "imports/c8s/rpm-4.14.3-21.el8": "a9e8d80332d37e00d3e07237cd042dcf248991de", - "imports/c8s/rpm-4.14.3-22.el8": "63d1365d625c4a6bcf3eff7b5488f5554ad4ca44", - "imports/c8s/rpm-4.14.3-3.el8": "35cbef94d0e7d64f3dbec7c0e4d533ebefd5f780", - "imports/c8s/rpm-4.14.3-4.el8": "09d0f647d30af7497fa2e9d45b1254280071fc7a", - "imports/c9-beta/rpm-4.16.1.3-11.el9": "137a597a82550d8c9563615e057ee6679e497ce6", - "imports/c9-beta/rpm-4.16.1.3-7.el9": "813822951aaa7e26eec176925250d9ce7430d62b", - "imports/c9-beta/rpm-4.16.1.3-9.el9": "3f3ddd0c8927e60ed546707bac1303a44035f0ec" - } - - tags2 = { - "imports/c8/rpm-4.14.3-13.el8": "b445f2ffef77c56fb45f6e679d1894d22867d501", - "imports/c8/rpm-4.14.3-14.el8_4": "41043c8511245c3abba654656d2161a2ad68791a", - "imports/c8/rpm-4.14.3-19.el8": "00810bfb118747fbe980b81845009dbfcfcc8450", - "imports/c8/rpm-4.14.3-19.el8_5.2": "377311e10f5d8b70af6d6d59a8d2e2e532893492", - "imports/c8/rpm-4.14.3-4.el8": "b7b8f7e8f5cdbef3992c8ee853ecc444df930503", - "imports/c8s/rpm-4.14.3-14.el8_4": "69c9a1638657acea0c70decddb0b76959032bbea", - "imports/c8s/rpm-4.14.3-15.el8": "5a7695fcfeb883271f76b96178c0fc9de9c3697d", - "imports/c8s/rpm-4.14.3-17.el8": "bfc6f7541d6e6229df6d7951d063c7631850cdcb", - "imports/c8s/rpm-4.14.3-18.el8": "d8c505ff117b10aa0905dd8567c031610a00c928", - "imports/c8s/rpm-4.14.3-19.el8": "dcdfdde122434aaba3986a8ed22056b072c9b9cd", - "imports/c8s/rpm-4.14.3-20.el8": "c7b760fdfae67300dc7d06ba3ad7fce7ad9a6c20", - "imports/c8s/rpm-4.14.3-21.el8": "a9e8d80332d37e00d3e07237cd042dcf248991de", - "imports/c8s/rpm-4.14.3-22.el8": "63d1365d625c4a6bcf3eff7b5488f5554ad4ca44", - "imports/c9-beta/rpm-4.16.1.3-11.el9": "137a597a82550d8c9563615e057ee6679e497ce6", - "imports/c9-beta/rpm-4.16.1.3-7.el9": "813822951aaa7e26eec176925250d9ce7430d62b", - "imports/c9-beta/rpm-4.16.1.3-9.el9": "3f3ddd0c8927e60ed546707bac1303a44035f0ec" - } - - tags3 = { - "imports/c8s/rpm-4.14.3-21.el8": "a9e8d80332d37e00d3e07237cd042dcf248991de", - "imports/c8s/rpm-4.14.3-22.el8": "63d1365d625c4a6bcf3eff7b5488f5554ad4ca44" - } - - tags4 = { - "imports/c8s/rpm-4.14.3-21.el8": "a9e8d80332d37e00d3e07237cd042dcf248991de" - } - - self.assertEqual(get_latest_version_with_commit('rpm', tags, 'c8s'), ('rpm-4.14.3-22.el8', '63d1365d625c4a6bcf3eff7b5488f5554ad4ca44')) - self.assertEqual(get_latest_version_with_commit('rpm', tags2, 'c8s'), ('rpm-4.14.3-22.el8', '63d1365d625c4a6bcf3eff7b5488f5554ad4ca44')) - self.assertEqual(get_latest_version_with_commit('rpm', tags3, 'c8s'), ('rpm-4.14.3-22.el8', '63d1365d625c4a6bcf3eff7b5488f5554ad4ca44')) - self.assertEqual(get_latest_version_with_commit('rpm', tags4, 'c8s'), ('rpm-4.14.3-21.el8', 'a9e8d80332d37e00d3e07237cd042dcf248991de')) - - - From 9ad4b2883f3dba245be434d10d5fa072a587c574 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Jul 18 2022 07:15:36 +0000 Subject: [PATCH 2/7] Add auto close logic --- diff --git a/package_updates/amqp/amqp.py b/package_updates/amqp/amqp.py index e69de29..131b710 100644 --- a/package_updates/amqp/amqp.py +++ b/package_updates/amqp/amqp.py @@ -0,0 +1,46 @@ +import sqlite3 +from .package_build import PackageBuild +from utils import helpers, db + +from fedora_messaging import api, config +from twisted import reactor + + +config.conf.setup_logging() + +CBS_BUILD_TOPIC = 'org.centos.prod.cbs.buildsys.build.state.change' + + +class AMQP: + def __init__(self, conn: sqlite3.Connection) -> None: + self.conn = conn + + def __issue_closer(self, build: PackageBuild) -> None: + centos_version = build.get_distro_version() + issue = helpers.get_open_issue(f'{build.name}.{centos_version}') + if issue: + build_url = build.get_build_url() + commit_url = build.get_commit_url() + comment = { + 'comment': f"""Build: {build_url} + Commit: {commit_url}""" + } + helpers.comment_on_issue(issue['id'], comment) + helpers.close_issue(issue['id']) + + def __on_message(self, msg): + print(str(msg)) + topic, body = msg.topic, msg.body + + build = PackageBuild(body) + saved_build_target = db.select_build_tag(build.build_target) + + if build.is_build_complete() and saved_build_target: + if topic == CBS_BUILD_TOPIC: + self.__issue_closer(build) + + def listen_on_amqp(self): + api.twisted_consume(self.__on_message) + reactor.run() + + diff --git a/package_updates/amqp/package_build.py b/package_updates/amqp/package_build.py new file mode 100644 index 0000000..0d0122a --- /dev/null +++ b/package_updates/amqp/package_build.py @@ -0,0 +1,35 @@ +import re +from utils.constants import * + + +class PackageBuild: + def __init__(self, body_msg: dict) -> None: + self.name = body_msg['name'] + self.version = body_msg['version'] + self.release = body_msg['release'] + self.build_id = body_msg['build_id'] + self.build_status = body_msg['new'] + self.source = body_msg['request'][0] + self.build_target = body_msg['request'][1] + self.package = ('%s-%s-%s' % (self.name, self.version, self.release)) + + def is_build_complete(self) -> bool: + if self.build_status == 1: + return True + return False + + def get_distro_version(self): + if 'el8' in self.release: + distro_version = 'c8s' + elif 'el9' in self.release: + distro_version = 'c9s' + else: + return None + return distro_version + + def get_build_url(self) -> str: + return f'{CBS_URL}buildinfo?buildID={self.build_id}' + + def get_commit_url(self) -> str: + url = re.findall(r'(https?://\S+)', self.source) + return url[0] \ No newline at end of file diff --git a/package_updates/main.py b/package_updates/main.py new file mode 100644 index 0000000..fe9b69e --- /dev/null +++ b/package_updates/main.py @@ -0,0 +1,110 @@ +#!/usr/bin/python + +import os +import sys +import random +import json +import time + +import paho.mqtt.client as mqtt + +from amqp.amqp import AMQP +from mqtt.mqtt import MQTT +from utils import helpers, db + + +H8S_MAIN_TAG_ID = 2249 # hyperscale8s-packages-main-release cbs koji tag +H8S_HOTFIXES_TAG_ID = 2305 # hyperscale8s-packages-hotfixes-release cbs koji tag +H8S_EXP_TAG_ID = 2245 # hyperscale8s-packages-experimental-release cbs koji tag +H8S_INTEL_CANDIDATE_TAG_ID = 2614 # hyperscale8s-packages-intel-candidate +H8S_INTEL_TESTING_TAG_ID = 2615 # hyperscale8s-packages-intel-testing +H8S_INTEL_RELEASE_TAG_ID = 2616 # hyperscale8s-packages-intel-release + +CBS_URL = 'https://cbs.centos.org/kojihub' +C8S_GIT_BRANCH = 'c8s' + +""" See https://wiki.centos.org/Sources for more details + for the following MQTT values. +""" +MQTT_BROKER = 'mqtt.git.centos.org' +MQTT_PORT = 8883 +TOPIC = 'git.centos.org/git.tag.creation' +CAFILE = os.environ['CAFILE'] +CERT = os.environ['CERT'] +KEY = os.environ['KEY'] +client_id = f'hyperscale-sig-{random.randint(0, 1000)}' +RUN_MODE = os.getenv('RUN_MODE', default='MQTT') + + +def setup(conn): + """Gets packages, creates database and sees if a package + update is available, for first time running. + """ + + tags = [H8S_MAIN_TAG_ID, H8S_HOTFIXES_TAG_ID, H8S_EXP_TAG_ID, + H8S_INTEL_CANDIDATE_TAG_ID, H8S_INTEL_TESTING_TAG_ID, H8S_INTEL_RELEASE_TAG_ID] + + session = helpers.get_koji_session(CBS_URL) + cbs_tags = helpers.get_koji_tags(session, tags) + packages = helpers.join_tagged_packages(session, tags) + builds = helpers.get_latest_tagged_builds(session, packages) + targets = helpers.get_koji_build_targets(session, 'hyperscale') #gets all hyperscale cbs build targets + + db.create_package_table(conn) + db.insert_packages_from_builds(builds, conn) + + db.create_tag_table(conn) + db.insert_tags(cbs_tags, conn) + + db.create_build_target_table(conn) + db.insert_build_targets(targets, conn) + + db.create_issue_table(conn) + + db_packages = db.select_packages(conn) + for package in db_packages: + package_name = package[1] + hs_version = ('%s-%s-%s' % (package[1], package[2], package[3])) + cbs_tag_id = package[5] + + centos_version, commit = helpers.get_git_pkg_version(package_name, C8S_GIT_BRANCH) + if centos_version == None: + continue + + print(f'centos version: {centos_version} | hs version: {hs_version}') + if helpers.compare_versions(centos_version, hs_version) == 1: + hs_tag_name = db.select_tag(cbs_tag_id, conn)[0] + issues = helpers.get_issues([package_name]) #gets all issues tagged with current package name + issue = helpers.issue_filter(package_name, issues) #gets the issue that is tagged with package name and package version + if issue: + version_tag = issue['version_tag'] + if centos_version != version_tag: + new_issue = helpers.create_ticket(package_name, centos_version, hs_version, hs_tag_name, commit) + helpers.comment_on_issue(issue['issue_id'], new_issue['id'], centos_version) + helpers.close_issue(issue['issue_id']) + db.insert_issue(new_issue['id'], package_name, centos_version, conn) + else: + db.insert_issue(issue['issue_id'], package_name, centos_version, conn) + else: + new_issue = helpers.create_ticket(package_name, centos_version, hs_version, hs_tag_name, commit) + db.insert_issue(new_issue['id'], package_name, centos_version, conn) + + +def main(): + print('running...') + try: + conn = db.get_connection() + setup(conn) + time.sleep(2) + if RUN_MODE == 'AMQP': + amqp_listener = AMQP(conn) + amqp_listener.listen_on_amqp() + else: + mqtt_listener = MQTT(conn) + mqtt_listener.listen_on_updates() + except KeyboardInterrupt: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/package_updates/mqtt/PackageRelease.py b/package_updates/mqtt/PackageRelease.py new file mode 100644 index 0000000..e673569 --- /dev/null +++ b/package_updates/mqtt/PackageRelease.py @@ -0,0 +1,33 @@ + + +class PackageRelease: + def __init__(self, payload: dict) -> None: + self.name = payload['repo']['name'] + self.git_tag = payload['tag'] + self.commit = payload['rev'] + self.build_id = body_msg['build_id'] + self.build_status = body_msg['new'] + self.source = body_msg['request'][0] + self.build_target = body_msg['request'][1] + self.package = ('%s-%s-%s' % (self.name, self.version, self.release)) + + def is_build_complete(self) -> bool: + if self.build_status == 1: + return True + return False + + def get_distro_version(self): + if 'el8' in self.release: + distro_version = 'c8s' + elif 'el9' in self.release: + distro_version = 'c9s' + else: + return None + return distro_version + + def get_build_url(self) -> str: + return f'{CBS_URL}buildinfo?buildID={self.build_id}' + + def get_commit_url(self) -> str: + url = re.findall(r'(https?://\S+)', self.source) + return url[0] \ No newline at end of file diff --git a/package_updates/mqtt/mqtt.py b/package_updates/mqtt/mqtt.py index e69de29..375a9a0 100644 --- a/package_updates/mqtt/mqtt.py +++ b/package_updates/mqtt/mqtt.py @@ -0,0 +1,79 @@ +import json +import sqlite3 + +import paho.mqtt.client as mqtt + +from utils import helpers, db +from .PackageRelease import PackageRelease + + +class MQTT: + def __init__(self, conn: sqlite3.Connection) -> None: + self.conn = conn + + def __on_connect(self, client, user_data, flags, rc): + """Callback, establishes a connection with the + mqtt server and subscribes to a specific topic. + """ + + if rc == 0: + print(f'Connected to MQTT Broker') + else: + print(f'Failed to connect, return code {rc}') + + client.subscribe(TOPIC) + + def __on_message(self, client, user_data, msg): + """Callback, when a message from mqtt is received deserializes the + payload and checks if it's a new package version to create an issue. + """ + + str_payload = msg.payload.decode('utf-8') + print(msg.topic + " " + str_payload) + payload = json.loads(str_payload) + + package_release = PackageRelease() + + pkg_name = package_release.name + git_tag = package_release.git_tag + commit = package_release.commit + + conn = user_data['db_conn'] + saved_hs_package = db.select_package(pkg_name, conn) + if saved_hs_package and C8S_GIT_BRANCH in git_tag: + self.__package_updater() + + def __package_updater(self): + session = helpers.get_koji_session(CBS_URL) + hs_tag_id = saved_hs_package[5] + cbs_hs_package = session.listTagged(hs_tag_id, latest=True, package=pkg_name) + hs_version = cbs_hs_package[0]['nvr'] + centos_version = helpers.filter_from_tag(git_tag, pkg_name) + + print(f'new version: {centos_version} | current version: {hs_version}') + if helpers.compare_versions(centos_version, hs_version) == 1: + issue = db.select_issue(pkg_name, conn) + hs_tag_name = db.select_tag(hs_tag_id, conn)[0] + new_issue = helpers.create_ticket(pkg_name, centos_version, hs_version, hs_tag_name, commit) + if issue: + db.update_issue_row(pkg_name, new_issue['id'], centos_version, conn) + is_open = helpers.is_issue_Open(centos_version) + if is_open: + helpers.comment_on_issue(issue[0], new_issue['id'], centos_version) + helpers.close_issue(issue[0]) + else: + db.insert_issue(new_issue['id'], pkg_name, centos_version, conn) + + def listen_on_updates(self): + """Sets up the mqtt client, starts the loop, and listens + on git.centos.org notifications for tag creation. + """ + + client = mqtt.Client() + client.on_connect = self.__on_connect + client.on_message = self.__on_message + client.tls_set(ca_certs=CAFILE, certfile=CERT, keyfile=KEY) + client.user_data_set({'db_conn': self.conn}) + client.connect(MQTT_BROKER, MQTT_PORT) + + client.loop_forever() \ No newline at end of file diff --git a/package_updates/utils/constants.py b/package_updates/utils/constants.py new file mode 100644 index 0000000..46f2f92 --- /dev/null +++ b/package_updates/utils/constants.py @@ -0,0 +1,8 @@ + +CBS_URL = 'https://cbs.centos.org/koji/' +KOJIHUB_URL = 'https://kojihub.stream.centos.org/koji/' + +PAGURE_REPO_API_URL = 'https://pagure.io/api/0/centos-sig-hyperscale/package-updates/' +PAGURE_REPO_URL = 'https://pagure.io/centos-sig-hyperscale/package-updates/' +GIT_CENTOS_API_URL = 'https://git.centos.org/api/0/rpms/' +GIT_CENTOS_URL = 'https://git.centos.org/rpms/' \ No newline at end of file diff --git a/package_updates/utils/db.py b/package_updates/utils/db.py index 695bf8c..3ec6d6c 100755 --- a/package_updates/utils/db.py +++ b/package_updates/utils/db.py @@ -24,6 +24,7 @@ def create_package_table(conn: sqlite3.Connection): name text, version text, release text, + distro_version, build_id INTEGER, tag_id INTEGER) """ @@ -53,13 +54,14 @@ def insert_package(package_id: int, name: str, version: str, release: str, version, release, build_id, + distro_version, tag_id) - VALUES(?, ?, ?, ?, ?, ?)""", + VALUES(?, ?, ?, ?, ?, ?, ?)""", (package_id, name, version, release, build_id, tag_id)) conn.commit() -def select_package(name: str, conn: sqlite3.Connection) -> tuple: +def select_package(name: str, distro_version: str, conn: sqlite3.Connection) -> tuple: """Gets a hs package from package table. Args: @@ -71,7 +73,7 @@ def select_package(name: str, conn: sqlite3.Connection) -> tuple: """ cursor = conn.cursor() - cursor.execute("SELECT * FROM package WHERE name=?", [name]) + cursor.execute("SELECT * FROM package WHERE name=? and distro_version=?", (name, distro_version)) return cursor.fetchone() diff --git a/package_updates/utils/helpers.py b/package_updates/utils/helpers.py index ddabf83..6c2fde4 100755 --- a/package_updates/utils/helpers.py +++ b/package_updates/utils/helpers.py @@ -105,6 +105,12 @@ def get_koji_tags(session: koji.ClientSession, tags_id: list) -> list: return tags +def get_koji_build_targets(session: koji.ClientSession, filter: str) -> list: + targets = session.getBuildTargets() + filtered_targets = [target for target in targets if filter in target['name']] + return filtered_targets + + def compare_versions(pkg1: str, pkg2: str) -> int: """Compares package version between two packages. @@ -293,6 +299,38 @@ def is_issue_Open(tag: str) -> bool: return False +def get_open_issue(tag: str) -> dict: + """Verify if the issue for a specific tag is open. + + Args: + tag: String of issue tag. + + Returns: + True if the issue for that package update + has been created, false otherwise. + """ + + params = { + 'status': 'Open', + 'tags': tag + } + url = urljoin(PAGURE_REPO_API_URL, 'issues') + + try: + res = requests.get(url=url, params=params) + res.raise_for_status() + except requests.exceptions.RequestException as err: + print(err) + return + + total_issues = res.json()['total_issues'] + if total_issues > 0: + issue = res.json()['issues'][0] + return issue + + return None + + def comment_on_issue(prev_issue_id: int, new_issue_id: int, version_tag: str): """Adds a comment to a specific pagure.io issue. From 48ff6263535831444d9c6ce14c50feb4f4cdde49 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Aug 19 2022 18:57:17 +0000 Subject: [PATCH 3/7] Separate constants --- diff --git a/package_updates/amqp/amqp.py b/package_updates/amqp/amqp.py index 131b710..e9b3b53 100644 --- a/package_updates/amqp/amqp.py +++ b/package_updates/amqp/amqp.py @@ -1,15 +1,14 @@ import sqlite3 -from .package_build import PackageBuild -from utils import helpers, db from fedora_messaging import api, config from twisted import reactor +from .package_build import PackageBuild +from utils import helpers, db +from settings.constants import CBS_BUILD_TOPIC -config.conf.setup_logging() - -CBS_BUILD_TOPIC = 'org.centos.prod.cbs.buildsys.build.state.change' +config.conf.setup_logging() class AMQP: def __init__(self, conn: sqlite3.Connection) -> None: diff --git a/package_updates/amqp/package_build.py b/package_updates/amqp/package_build.py index 0d0122a..0ede793 100644 --- a/package_updates/amqp/package_build.py +++ b/package_updates/amqp/package_build.py @@ -1,5 +1,5 @@ import re -from utils.constants import * +from ..settings.constants import CBS_URL class PackageBuild: diff --git a/package_updates/main.py b/package_updates/main.py index fe9b69e..c44b10b 100644 --- a/package_updates/main.py +++ b/package_updates/main.py @@ -1,9 +1,6 @@ #!/usr/bin/python -import os import sys -import random -import json import time import paho.mqtt.client as mqtt @@ -11,29 +8,7 @@ import paho.mqtt.client as mqtt from amqp.amqp import AMQP from mqtt.mqtt import MQTT from utils import helpers, db - - -H8S_MAIN_TAG_ID = 2249 # hyperscale8s-packages-main-release cbs koji tag -H8S_HOTFIXES_TAG_ID = 2305 # hyperscale8s-packages-hotfixes-release cbs koji tag -H8S_EXP_TAG_ID = 2245 # hyperscale8s-packages-experimental-release cbs koji tag -H8S_INTEL_CANDIDATE_TAG_ID = 2614 # hyperscale8s-packages-intel-candidate -H8S_INTEL_TESTING_TAG_ID = 2615 # hyperscale8s-packages-intel-testing -H8S_INTEL_RELEASE_TAG_ID = 2616 # hyperscale8s-packages-intel-release - -CBS_URL = 'https://cbs.centos.org/kojihub' -C8S_GIT_BRANCH = 'c8s' - -""" See https://wiki.centos.org/Sources for more details - for the following MQTT values. -""" -MQTT_BROKER = 'mqtt.git.centos.org' -MQTT_PORT = 8883 -TOPIC = 'git.centos.org/git.tag.creation' -CAFILE = os.environ['CAFILE'] -CERT = os.environ['CERT'] -KEY = os.environ['KEY'] -client_id = f'hyperscale-sig-{random.randint(0, 1000)}' -RUN_MODE = os.getenv('RUN_MODE', default='MQTT') +from settings.constants import HS_CBS_TAGS, CBS_URL, C8S_GIT_BRANCH, RUN_MODE def setup(conn): @@ -41,8 +16,7 @@ def setup(conn): update is available, for first time running. """ - tags = [H8S_MAIN_TAG_ID, H8S_HOTFIXES_TAG_ID, H8S_EXP_TAG_ID, - H8S_INTEL_CANDIDATE_TAG_ID, H8S_INTEL_TESTING_TAG_ID, H8S_INTEL_RELEASE_TAG_ID] + tags = HS_CBS_TAGS session = helpers.get_koji_session(CBS_URL) cbs_tags = helpers.get_koji_tags(session, tags) diff --git a/package_updates/mqtt/PackageRelease.py b/package_updates/mqtt/PackageRelease.py deleted file mode 100644 index e673569..0000000 --- a/package_updates/mqtt/PackageRelease.py +++ /dev/null @@ -1,33 +0,0 @@ - - -class PackageRelease: - def __init__(self, payload: dict) -> None: - self.name = payload['repo']['name'] - self.git_tag = payload['tag'] - self.commit = payload['rev'] - self.build_id = body_msg['build_id'] - self.build_status = body_msg['new'] - self.source = body_msg['request'][0] - self.build_target = body_msg['request'][1] - self.package = ('%s-%s-%s' % (self.name, self.version, self.release)) - - def is_build_complete(self) -> bool: - if self.build_status == 1: - return True - return False - - def get_distro_version(self): - if 'el8' in self.release: - distro_version = 'c8s' - elif 'el9' in self.release: - distro_version = 'c9s' - else: - return None - return distro_version - - def get_build_url(self) -> str: - return f'{CBS_URL}buildinfo?buildID={self.build_id}' - - def get_commit_url(self) -> str: - url = re.findall(r'(https?://\S+)', self.source) - return url[0] \ No newline at end of file diff --git a/package_updates/mqtt/mqtt.py b/package_updates/mqtt/mqtt.py index 375a9a0..9ed74e6 100644 --- a/package_updates/mqtt/mqtt.py +++ b/package_updates/mqtt/mqtt.py @@ -4,12 +4,14 @@ import sqlite3 import paho.mqtt.client as mqtt from utils import helpers, db -from .PackageRelease import PackageRelease +from .package_git_release import PackageGitRelease +from settings.constants import C8S_GIT_BRANCH, CBS_URL, CAFILE, CERT, KEY, MQTT_BROKER, MQTT_PORT class MQTT: - def __init__(self, conn: sqlite3.Connection) -> None: - self.conn = conn + def __init__(self, db_conn: sqlite3.Connection, topic: str) -> None: + self.db_conn = db_conn + self.topic = topic def __on_connect(self, client, user_data, flags, rc): """Callback, establishes a connection with the @@ -21,7 +23,8 @@ class MQTT: else: print(f'Failed to connect, return code {rc}') - client.subscribe(TOPIC) + client.subscribe(self.topic) + def __on_message(self, client, user_data, msg): """Callback, when a message from mqtt is received deserializes the @@ -32,18 +35,22 @@ class MQTT: print(msg.topic + " " + str_payload) payload = json.loads(str_payload) - package_release = PackageRelease() + package_release = PackageGitRelease(payload) pkg_name = package_release.name git_tag = package_release.git_tag - commit = package_release.commit - conn = user_data['db_conn'] - saved_hs_package = db.select_package(pkg_name, conn) + db_conn = user_data['db_conn'] + saved_hs_package = db.select_package(pkg_name, db_conn) if saved_hs_package and C8S_GIT_BRANCH in git_tag: - self.__package_updater() + self.__package_updater(package_release, saved_hs_package) - def __package_updater(self): + + def __package_updater(self, package_release: PackageGitRelease, saved_hs_package): + pkg_name = package_release.name + git_tag = package_release.git_tag + commit = package_release.commit + session = helpers.get_koji_session(CBS_URL) hs_tag_id = saved_hs_package[5] cbs_hs_package = session.listTagged(hs_tag_id, latest=True, package=pkg_name) @@ -52,17 +59,18 @@ class MQTT: print(f'new version: {centos_version} | current version: {hs_version}') if helpers.compare_versions(centos_version, hs_version) == 1: - issue = db.select_issue(pkg_name, conn) - hs_tag_name = db.select_tag(hs_tag_id, conn)[0] + issue = db.select_issue(pkg_name, self.db_conn) + hs_tag_name = db.select_tag(hs_tag_id, self.db_conn)[0] new_issue = helpers.create_ticket(pkg_name, centos_version, hs_version, hs_tag_name, commit) if issue: - db.update_issue_row(pkg_name, new_issue['id'], centos_version, conn) + db.update_issue_row(pkg_name, new_issue['id'], centos_version, self.db_conn) is_open = helpers.is_issue_Open(centos_version) if is_open: helpers.comment_on_issue(issue[0], new_issue['id'], centos_version) helpers.close_issue(issue[0]) else: - db.insert_issue(new_issue['id'], pkg_name, centos_version, conn) + db.insert_issue(new_issue['id'], pkg_name, centos_version, self.db_conn) + def listen_on_updates(self): """Sets up the mqtt client, starts the loop, and listens @@ -73,7 +81,7 @@ class MQTT: client.on_connect = self.__on_connect client.on_message = self.__on_message client.tls_set(ca_certs=CAFILE, certfile=CERT, keyfile=KEY) - client.user_data_set({'db_conn': self.conn}) + client.user_data_set({'db_conn': self.db_conn}) client.connect(MQTT_BROKER, MQTT_PORT) client.loop_forever() \ No newline at end of file diff --git a/package_updates/mqtt/package_git_release.py b/package_updates/mqtt/package_git_release.py new file mode 100644 index 0000000..f4de39c --- /dev/null +++ b/package_updates/mqtt/package_git_release.py @@ -0,0 +1,10 @@ + +class PackageGitRelease: + def __init__(self, payload: dict) -> None: + self.name = payload['repo']['name'] + self.git_tag = payload['tag'] + self.commit = payload['rev'] + + def get_package_version(self): + index = self.git_tag.find(self.name) + return self.git_tag[index:] diff --git a/package_updates/settings/constants.py b/package_updates/settings/constants.py new file mode 100644 index 0000000..1350958 --- /dev/null +++ b/package_updates/settings/constants.py @@ -0,0 +1,48 @@ +import os +import random + + +# URLS +CBS_URL = 'https://cbs.centos.org/koji/' +KOJIHUB_URL = 'https://kojihub.stream.centos.org/koji/' +PAGURE_REPO_API_URL = 'https://pagure.io/api/0/centos-sig-hyperscale/package-updates/' +PAGURE_REPO_URL = 'https://pagure.io/centos-sig-hyperscale/package-updates/' +GIT_CENTOS_API_URL = 'https://git.centos.org/api/0/rpms/' +GIT_CENTOS_URL = 'https://git.centos.org/rpms/' + + +# Git branches +C8S_GIT_BRANCH = 'c8s' + + +# MQTT +""" See https://wiki.centos.org/Sources for more details + for the following MQTT values. +""" +MQTT_BROKER = 'mqtt.git.centos.org' +MQTT_PORT = 8883 +TOPIC = 'git.centos.org/git.tag.creation' +CAFILE = os.environ['CAFILE'] +CERT = os.environ['CERT'] +KEY = os.environ['KEY'] +client_id = f'hyperscale-sig-{random.randint(0, 1000)}' +RUN_MODE = os.getenv('RUN_MODE', default='MQTT') + + +# AMQP +CBS_BUILD_TOPIC = 'org.centos.prod.cbs.buildsys.build.state.change' + + +# CBS tags +H8S_MAIN_TAG_ID = 2249 # hyperscale8s-packages-main-release cbs koji tag +H8S_HOTFIXES_TAG_ID = 2305 # hyperscale8s-packages-hotfixes-release cbs koji tag +H8S_EXP_TAG_ID = 2245 # hyperscale8s-packages-experimental-release cbs koji tag +H8S_INTEL_CANDIDATE_TAG_ID = 2614 # hyperscale8s-packages-intel-candidate +H8S_INTEL_TESTING_TAG_ID = 2615 # hyperscale8s-packages-intel-testing +H8S_INTEL_RELEASE_TAG_ID = 2616 # hyperscale8s-packages-intel-release +HS_CBS_TAGS = [H8S_MAIN_TAG_ID, H8S_HOTFIXES_TAG_ID, H8S_EXP_TAG_ID, + H8S_INTEL_CANDIDATE_TAG_ID, H8S_INTEL_TESTING_TAG_ID, H8S_INTEL_RELEASE_TAG_ID] + + +# Run mode +RUN_MODE = os.getenv('RUN_MODE', 'MQTT') \ No newline at end of file diff --git a/package_updates/utils/constants.py b/package_updates/utils/constants.py deleted file mode 100644 index 46f2f92..0000000 --- a/package_updates/utils/constants.py +++ /dev/null @@ -1,8 +0,0 @@ - -CBS_URL = 'https://cbs.centos.org/koji/' -KOJIHUB_URL = 'https://kojihub.stream.centos.org/koji/' - -PAGURE_REPO_API_URL = 'https://pagure.io/api/0/centos-sig-hyperscale/package-updates/' -PAGURE_REPO_URL = 'https://pagure.io/centos-sig-hyperscale/package-updates/' -GIT_CENTOS_API_URL = 'https://git.centos.org/api/0/rpms/' -GIT_CENTOS_URL = 'https://git.centos.org/rpms/' \ No newline at end of file diff --git a/package_updates/utils/helpers.py b/package_updates/utils/helpers.py index 6c2fde4..70fbea8 100755 --- a/package_updates/utils/helpers.py +++ b/package_updates/utils/helpers.py @@ -5,11 +5,8 @@ import hawkey import koji import requests +from ..settings.constants import GIT_CENTOS_API_URL, GIT_CENTOS_URL, PAGURE_REPO_API_URL, PAGURE_REPO_URL -PAGURE_REPO_API_URL = 'https://pagure.io/api/0/centos-sig-hyperscale/package-updates/' -PAGURE_REPO_URL = 'https://pagure.io/centos-sig-hyperscale/package-updates/' -GIT_CENTOS_API_URL = 'https://git.centos.org/api/0/rpms/' -GIT_CENTOS_URL = 'https://git.centos.org/rpms/' def get_koji_session(url: str) -> koji.ClientSession: """Connects to the koji build system. From 48d2a0b9abe8700de83a28bb575c764280bd4218 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sep 09 2022 17:36:29 +0000 Subject: [PATCH 4/7] Modify DB functions & fix script --- diff --git a/Dockerfile b/Dockerfile index 6d9e9f8..2fcaa8c 100755 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,9 @@ RUN dnf update -y && \ dnf install -y \ gcc \ krb5-devel \ - python3-devel + python3-devel \ + vim \ + wget RUN dnf clean all @@ -13,14 +15,17 @@ RUN useradd -r -d $HOME app RUN usermod -aG wheel app WORKDIR /home/app/package-updates -RUN mkdir utils - RUN chown -R app:wheel $HOME/package-updates + +ADD --chown=app:wheel https://raw.githubusercontent.com/fedora-infra/fedora-messaging/stable/configs/fedora-key.pem $HOME/package-updates/ +ADD --chown=app:wheel https://raw.githubusercontent.com/fedora-infra/fedora-messaging/stable/configs/fedora-cert.pem $HOME/package-updates/ +ADD --chown=app:wheel https://raw.githubusercontent.com/fedora-infra/fedora-messaging/stable/configs/cacert.pem $HOME/package-updates/ + COPY --chown=app:wheel ./package_updates/ $HOME/package-updates/ -RUN ls -la $HOME/package-updates/* -COPY --chown=app:wheel ./requirements.txt $HOME/package-updates +COPY --chown=app:wheel ./fedora.toml $HOME/package-updates/ +COPY --chown=app:wheel ./requirements.txt $HOME/package-updates/ RUN pip3 install -r requirements.txt -ENTRYPOINT ["python3", "./package_updates.py"] +ENTRYPOINT ["python3", "./main.py"] USER app diff --git a/fedora.toml b/fedora.toml new file mode 100644 index 0000000..9e381f1 --- /dev/null +++ b/fedora.toml @@ -0,0 +1,96 @@ +# A basic configuration for Fedora's message broker, using the example callback +# which simply prints messages to standard output. +# +# This file is in the TOML format. +amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub" +callback = "fedora_messaging.example:printer" + +[tls] +ca_cert = "/home/app/package-updates/cacert.pem" +keyfile = "/home/app/package-updates/fedora-key.pem" +certfile = "/home/app/package-updates/fedora-cert.pem" + +[client_properties] +app = "Example Application" +# Some suggested extra fields: +# URL of the project that provides this consumer +app_url = "https://github.com/fedora-infra/fedora-messaging" +# Contact emails for the maintainer(s) of the consumer - in case the +# broker admin needs to contact them, for e.g. +app_contacts_email = ["admin@fedoraproject.org"] + +[exchanges."amq.topic"] +type = "topic" +durable = true +auto_delete = false +arguments = {} + +# Queue names *must* be in the normal UUID format: run "uuidgen" and use the +# output as your queue name. If you don't define a queue here, the server will +# generate a queue name for you. This queue will be non-durable, auto-deleted and +# exclusive. +# If your queue is not exclusive, anyone can connect and consume from it, causing +# you to miss messages, so do not share your queue name. Any queues that are not +# auto-deleted on disconnect are garbage-collected after approximately one hour. +# +# If you require a stronger guarantee about delivery, please talk to Fedora's +# Infrastructure team. +# +# [queues.00000000-0000-0000-0000-000000000000] +# durable = false +# auto_delete = true +# exclusive = true +# arguments = {} + +# If you use the server-generated queue names, you can leave out the "queue" +# parameter in the bindings definition. +[[bindings]] +# queue = "00000000-0000-0000-0000-000000000000" +exchange = "amq.topic" +routing_keys = ["org.centos.prod.cbs.buildsys.build.state.change"] # Set this to the specific topics you are interested in. + +[consumer_config] +example_key = "for my consumer" + +[qos] +prefetch_size = 0 +prefetch_count = 25 + +[log_config] +version = 1 +disable_existing_loggers = true + +[log_config.formatters.simple] +format = "[%(levelname)s %(name)s] %(message)s" + +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" + +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.twisted] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.pika] +level = "WARNING" +propagate = false +handlers = ["console"] + +# If your consumer sets up a logger, you must add a configuration for it +# here in order for the messages to show up. e.g. if it set up a logger +# called 'example_printer', you could do: +#[log_config.loggers.example_printer] +#level = "INFO" +#propagate = false +#handlers = ["console"] + +[log_config.root] +level = "ERROR" +handlers = ["console"] \ No newline at end of file diff --git a/package_updates/amqp/amqp.py b/package_updates/amqp/amqp.py index e9b3b53..1589af9 100644 --- a/package_updates/amqp/amqp.py +++ b/package_updates/amqp/amqp.py @@ -1,21 +1,28 @@ import sqlite3 from fedora_messaging import api, config -from twisted import reactor +from twisted.internet import reactor from .package_build import PackageBuild -from utils import helpers, db +from utils import helpers +from utils.db import BuildTargetDB from settings.constants import CBS_BUILD_TOPIC config.conf.setup_logging() + class AMQP: - def __init__(self, conn: sqlite3.Connection) -> None: - self.conn = conn + """Class for AMQP Run mode. + """ + + def __init__(self, db_conn: sqlite3.Connection) -> None: + self.db_conn = db_conn def __issue_closer(self, build: PackageBuild) -> None: - centos_version = build.get_distro_version() + """Auto close an issue for a specific package if it's open. + """ + centos_version = helpers.get_centos_version(build.release) issue = helpers.get_open_issue(f'{build.name}.{centos_version}') if issue: build_url = build.get_build_url() @@ -25,20 +32,27 @@ class AMQP: Commit: {commit_url}""" } helpers.comment_on_issue(issue['id'], comment) - helpers.close_issue(issue['id']) + helpers.close_issue(issue['id'], 'Fixed') def __on_message(self, msg): + """Callback. Called when a message is resieved by fedora messaging, checks if a + package build from cbs is complete and if the build target is saved in the DB. + """ + print(str(msg)) topic, body = msg.topic, msg.body - + build = PackageBuild(body) - saved_build_target = db.select_build_tag(build.build_target) + build_target_db = BuildTargetDB(self.db_conn) + saved_build_target = build_target_db.select_build_target(build.build_target) if build.is_build_complete() and saved_build_target: if topic == CBS_BUILD_TOPIC: self.__issue_closer(build) def listen_on_amqp(self): + """Connects to fedora messaging AMQP broker and keeps listening + """ api.twisted_consume(self.__on_message) reactor.run() diff --git a/package_updates/amqp/package_build.py b/package_updates/amqp/package_build.py index 0ede793..a0172de 100644 --- a/package_updates/amqp/package_build.py +++ b/package_updates/amqp/package_build.py @@ -1,8 +1,13 @@ import re -from ..settings.constants import CBS_URL +from settings.constants import CBS_BUILD_URL class PackageBuild: + """Represents a package build entity from the AMQP + #buildsys.build.state.change topics payload. + Gets only necessary values from the payload. + """ + def __init__(self, body_msg: dict) -> None: self.name = body_msg['name'] self.version = body_msg['version'] @@ -14,22 +19,22 @@ class PackageBuild: self.package = ('%s-%s-%s' % (self.name, self.version, self.release)) def is_build_complete(self) -> bool: + """Checks if a package build from koji is complete. + """ + if self.build_status == 1: return True return False - def get_distro_version(self): - if 'el8' in self.release: - distro_version = 'c8s' - elif 'el9' in self.release: - distro_version = 'c9s' - else: - return None - return distro_version - def get_build_url(self) -> str: - return f'{CBS_URL}buildinfo?buildID={self.build_id}' + """Generates a url for a koji package build. + """ + + return f'{CBS_BUILD_URL}{self.build_id}' def get_commit_url(self) -> str: + """Gets the commit url from the #buildsys.build.state.change topic payload. + """ + url = re.findall(r'(https?://\S+)', self.source) return url[0] \ No newline at end of file diff --git a/package_updates/main.py b/package_updates/main.py index c44b10b..43d4056 100644 --- a/package_updates/main.py +++ b/package_updates/main.py @@ -2,13 +2,12 @@ import sys import time - -import paho.mqtt.client as mqtt +from urllib.parse import urljoin from amqp.amqp import AMQP from mqtt.mqtt import MQTT from utils import helpers, db -from settings.constants import HS_CBS_TAGS, CBS_URL, C8S_GIT_BRANCH, RUN_MODE +from settings.constants import HS_CBS_TAGS, CBS_URL, C8S_GIT_BRANCH, RUN_MODE, PAGURE_REPO_URL def setup(conn): @@ -20,48 +19,58 @@ def setup(conn): session = helpers.get_koji_session(CBS_URL) cbs_tags = helpers.get_koji_tags(session, tags) - packages = helpers.join_tagged_packages(session, tags) - builds = helpers.get_latest_tagged_builds(session, packages) + builds = helpers.get_latest_tagged_builds(session, tags) targets = helpers.get_koji_build_targets(session, 'hyperscale') #gets all hyperscale cbs build targets - db.create_package_table(conn) - db.insert_packages_from_builds(builds, conn) + tag_db = db.TagDB(conn) + tag_db.create_tag_table() + tag_db.insert_tags(cbs_tags) - db.create_tag_table(conn) - db.insert_tags(cbs_tags, conn) + package_db = db.PackageDB(conn) + package_db.create_package_table() + package_db.insert_packages_from_builds(builds) - db.create_build_target_table(conn) - db.insert_build_targets(targets, conn) + build_target_db = db.BuildTargetDB(conn) + build_target_db.create_build_target_table() + build_target_db.insert_build_targets(targets) - db.create_issue_table(conn) + issue_db = db.IssueDB(conn) + issue_db.create_issue_table() - db_packages = db.select_packages(conn) - for package in db_packages: - package_name = package[1] - hs_version = ('%s-%s-%s' % (package[1], package[2], package[3])) - cbs_tag_id = package[5] + saved_packages = package_db.select_packages() + for package in saved_packages: + package_name = package['name'] + hs_version = ('%s-%s-%s' % (package['name'], package['version'], package['release'])) + cbs_tag_id = package['tag_id'] - centos_version, commit = helpers.get_git_pkg_version(package_name, C8S_GIT_BRANCH) - if centos_version == None: + upstream_pkg_version, commit = helpers.get_git_pkg_version(package_name, C8S_GIT_BRANCH) + if upstream_pkg_version == None: continue - print(f'centos version: {centos_version} | hs version: {hs_version}') - if helpers.compare_versions(centos_version, hs_version) == 1: - hs_tag_name = db.select_tag(cbs_tag_id, conn)[0] + print(f'centos version: {upstream_pkg_version} | hs version: {hs_version}') + if helpers.compare_versions(upstream_pkg_version, hs_version) == 1: + centos_version = helpers.get_centos_version(upstream_pkg_version) + hs_tag_name = tag_db.select_tag(cbs_tag_id)['tag_name'] issues = helpers.get_issues([package_name]) #gets all issues tagged with current package name - issue = helpers.issue_filter(package_name, issues) #gets the issue that is tagged with package name and package version + issue = helpers.issue_filter(issues, package_name) #gets the issue that is tagged with package name and package version if issue: version_tag = issue['version_tag'] - if centos_version != version_tag: - new_issue = helpers.create_ticket(package_name, centos_version, hs_version, hs_tag_name, commit) - helpers.comment_on_issue(issue['issue_id'], new_issue['id'], centos_version) - helpers.close_issue(issue['issue_id']) - db.insert_issue(new_issue['id'], package_name, centos_version, conn) + if upstream_pkg_version != version_tag: + new_issue = helpers.create_ticket(package_name, upstream_pkg_version, hs_version, hs_tag_name, commit, centos_version) + new_issue_url = urljoin(PAGURE_REPO_URL, f"issue/{new_issue['id']}") + comment = { + 'comment': f"""{upstream_pkg_version} is available and this issue still open. + New issue has been created for the newer version. + URL: {new_issue_url}""" + } + helpers.comment_on_issue(issue['issue_id'], comment) + helpers.close_issue(issue['issue_id'], 'Invalid') + issue_db.insert_issue(new_issue['id'], package['package_id']) else: - db.insert_issue(issue['issue_id'], package_name, centos_version, conn) + issue_db.insert_issue(issue['issue_id'], package['package_id']) else: - new_issue = helpers.create_ticket(package_name, centos_version, hs_version, hs_tag_name, commit) - db.insert_issue(new_issue['id'], package_name, centos_version, conn) + new_issue = helpers.create_ticket(package_name, upstream_pkg_version, hs_version, hs_tag_name, commit, centos_version) + issue_db.insert_issue(new_issue['id'], package['package_id']) def main(): diff --git a/package_updates/mqtt/mqtt.py b/package_updates/mqtt/mqtt.py index 9ed74e6..1823d32 100644 --- a/package_updates/mqtt/mqtt.py +++ b/package_updates/mqtt/mqtt.py @@ -1,17 +1,21 @@ import json import sqlite3 +from urllib.parse import urljoin import paho.mqtt.client as mqtt -from utils import helpers, db +from utils import helpers +from utils.db import PackageDB, IssueDB, TagDB from .package_git_release import PackageGitRelease -from settings.constants import C8S_GIT_BRANCH, CBS_URL, CAFILE, CERT, KEY, MQTT_BROKER, MQTT_PORT +from settings.constants import C8S_GIT_BRANCH, CBS_URL, CAFILE, CERT, KEY, MQTT_BROKER, MQTT_PORT, PAGURE_REPO_URL, MQTT_TOPIC class MQTT: - def __init__(self, db_conn: sqlite3.Connection, topic: str) -> None: + """Class for MQTT run mode. + """ + + def __init__(self, db_conn: sqlite3.Connection) -> None: self.db_conn = db_conn - self.topic = topic def __on_connect(self, client, user_data, flags, rc): """Callback, establishes a connection with the @@ -23,8 +27,7 @@ class MQTT: else: print(f'Failed to connect, return code {rc}') - client.subscribe(self.topic) - + client.subscribe(MQTT_TOPIC) def __on_message(self, client, user_data, msg): """Callback, when a message from mqtt is received deserializes the @@ -37,44 +40,51 @@ class MQTT: package_release = PackageGitRelease(payload) - pkg_name = package_release.name - git_tag = package_release.git_tag - db_conn = user_data['db_conn'] - saved_hs_package = db.select_package(pkg_name, db_conn) - if saved_hs_package and C8S_GIT_BRANCH in git_tag: + pkg_db = PackageDB(db_conn) + saved_hs_package = pkg_db.select_package(package_release.name, C8S_GIT_BRANCH) + if saved_hs_package and C8S_GIT_BRANCH in package_release.git_tag: self.__package_updater(package_release, saved_hs_package) - def __package_updater(self, package_release: PackageGitRelease, saved_hs_package): + """Creates in issue for a package update. + """ pkg_name = package_release.name - git_tag = package_release.git_tag - commit = package_release.commit + + issue_db = IssueDB(self.db_conn) + tag_db = TagDB(self.db_conn) session = helpers.get_koji_session(CBS_URL) - hs_tag_id = saved_hs_package[5] + hs_tag_id = saved_hs_package['tag_id'] cbs_hs_package = session.listTagged(hs_tag_id, latest=True, package=pkg_name) hs_version = cbs_hs_package[0]['nvr'] - centos_version = helpers.filter_from_tag(git_tag, pkg_name) + centos_version = helpers.filter_from_tag(package_release.git_tag, pkg_name) print(f'new version: {centos_version} | current version: {hs_version}') if helpers.compare_versions(centos_version, hs_version) == 1: - issue = db.select_issue(pkg_name, self.db_conn) - hs_tag_name = db.select_tag(hs_tag_id, self.db_conn)[0] - new_issue = helpers.create_ticket(pkg_name, centos_version, hs_version, hs_tag_name, commit) + issue = issue_db.select_issue(saved_hs_package['package_id']) + hs_tag_name = tag_db.select_tag(hs_tag_id)['tag_name'] + is_open = helpers.is_issue_Open(f'{pkg_name}.c8s') + new_issue = helpers.create_ticket(pkg_name, centos_version, hs_version, hs_tag_name, package_release.commit, C8S_GIT_BRANCH) if issue: - db.update_issue_row(pkg_name, new_issue['id'], centos_version, self.db_conn) - is_open = helpers.is_issue_Open(centos_version) + print('issueee1') + issue_db.update_issue_row(new_issue['id'], saved_hs_package['package_id']) if is_open: - helpers.comment_on_issue(issue[0], new_issue['id'], centos_version) - helpers.close_issue(issue[0]) + new_issue_url = urljoin(PAGURE_REPO_URL, f"issue/{new_issue['id']}") + comment = { + 'comment': f"""{centos_version} is available and this issue still open. + New issue has been created for the newer version. + URL: {new_issue_url}""" + } + helpers.comment_on_issue(issue['issue_id'], comment) + helpers.close_issue(issue['issue_id'], 'Invalid') else: - db.insert_issue(new_issue['id'], pkg_name, centos_version, self.db_conn) - + print('no issuee') + issue_db.insert_issue(new_issue['id'], saved_hs_package['package_id']) def listen_on_updates(self): """Sets up the mqtt client, starts the loop, and listens - on git.centos.org notifications for tag creation. + on git.centos.org notifications for tag creation. """ client = mqtt.Client() diff --git a/package_updates/mqtt/package_git_release.py b/package_updates/mqtt/package_git_release.py index f4de39c..6fe0328 100644 --- a/package_updates/mqtt/package_git_release.py +++ b/package_updates/mqtt/package_git_release.py @@ -1,10 +1,11 @@ class PackageGitRelease: + """Represents a package git release tag from the MQTT + git.centos.org/git.tag.creation topic payload. + Gets only necessary values from the payload. + """ + def __init__(self, payload: dict) -> None: self.name = payload['repo']['name'] self.git_tag = payload['tag'] self.commit = payload['rev'] - - def get_package_version(self): - index = self.git_tag.find(self.name) - return self.git_tag[index:] diff --git a/package_updates/package_updates.py b/package_updates/package_updates.py deleted file mode 100755 index 1e7ea7d..0000000 --- a/package_updates/package_updates.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/python - -import os -import sys -import random -import json -import time - -import paho.mqtt.client as mqtt - -from utils import helpers, db - - -H8S_MAIN_TAG_ID = 2249 # hyperscale8s-packages-main-release cbs koji tag -H8S_HOTFIXES_TAG_ID = 2305 # hyperscale8s-packages-hotfixes-release cbs koji tag -H8S_EXP_TAG_ID = 2245 # hyperscale8s-packages-experimental-release cbs koji tag -H8S_INTEL_CANDIDATE_TAG_ID = 2614 # hyperscale8s-packages-intel-candidate -H8S_INTEL_TESTING_TAG_ID = 2615 # hyperscale8s-packages-intel-testing -H8S_INTEL_RELEASE_TAG_ID = 2616 # hyperscale8s-packages-intel-release - -CBS_URL = 'https://cbs.centos.org/kojihub' -C8S_GIT_BRANCH = 'c8s' - -""" See https://wiki.centos.org/Sources for more details - for the following MQTT values. -""" -MQTT_BROKER = 'mqtt.git.centos.org' -MQTT_PORT = 8883 -TOPIC = 'git.centos.org/git.tag.creation' -CAFILE = os.environ['CAFILE'] -CERT = os.environ['CERT'] -KEY = os.environ['KEY'] -client_id = f'hyperscale-sig-{random.randint(0, 1000)}' - - -def setup(conn): - """Gets packages, creates database and sees if a package - update is available, for first time running. - """ - - tags = [H8S_MAIN_TAG_ID, H8S_HOTFIXES_TAG_ID, H8S_EXP_TAG_ID, - H8S_INTEL_CANDIDATE_TAG_ID, H8S_INTEL_TESTING_TAG_ID, H8S_INTEL_RELEASE_TAG_ID] - - session = helpers.get_koji_session(CBS_URL) - cbs_tags = helpers.get_koji_tags(session, tags) - packages = helpers.join_tagged_packages(session, tags) - builds = helpers.get_latest_tagged_builds(session, packages) - - db.create_package_table(conn) - db.insert_packages_from_builds(builds, conn) - - db.create_tag_table(conn) - db.insert_tags(cbs_tags, conn) - - db.create_issue_table(conn) - - db_packages = db.select_packages(conn) - for package in db_packages: - package_name = package[1] - hs_version = ('%s-%s-%s' % (package[1], package[2], package[3])) - cbs_tag_id = package[5] - - centos_version, commit = helpers.get_git_pkg_version(package_name, C8S_GIT_BRANCH) - if centos_version == None: - continue - - print(f'centos version: {centos_version} | hs version: {hs_version}') - if helpers.compare_versions(centos_version, hs_version) == 1: - hs_tag_name = db.select_tag(cbs_tag_id, conn)[0] - issues = helpers.get_issues([package_name]) #gets all issues tagged with current package name - issue = helpers.issue_filter(package_name, issues) #gets the issue that is tagged with package name and package version - if issue: - version_tag = issue['version_tag'] - if centos_version != version_tag: - new_issue = helpers.create_ticket(package_name, centos_version, hs_version, hs_tag_name, commit) - helpers.comment_on_issue(issue['issue_id'], new_issue['id'], centos_version) - helpers.close_issue(issue['issue_id']) - db.insert_issue(new_issue['id'], package_name, centos_version, conn) - else: - db.insert_issue(issue['issue_id'], package_name, centos_version, conn) - else: - new_issue = helpers.create_ticket(package_name, centos_version, hs_version, hs_tag_name, commit) - db.insert_issue(new_issue['id'], package_name, centos_version, conn) - - -def on_connect(client, user_data, flags, rc): - """Callback, establishes a connection with the - mqtt server and subscribes to a specific topic. - """ - - if rc == 0: - print(f'Connected to MQTT Broker') - else: - print(f'Failed to connect, return code {rc}') - - client.subscribe(TOPIC) - - -def on_message(client, user_data, msg): - """Callback, when a message from mqtt is received deserializes the - payload and checks if it's a new package version to create an issue. - """ - - str_payload = msg.payload.decode('utf-8') - print(msg.topic + " " + str_payload) - payload = json.loads(str_payload) - - pkg_name = payload['repo']['name'] - git_tag = payload['tag'] - commit = payload['rev'] - - conn = user_data['db_conn'] - saved_hs_package = db.select_package(pkg_name, conn) - if saved_hs_package and C8S_GIT_BRANCH in git_tag: - session = helpers.get_koji_session(CBS_URL) - hs_tag_id = saved_hs_package[5] - cbs_hs_package = session.listTagged(hs_tag_id, latest=True, package=pkg_name) - hs_version = cbs_hs_package[0]['nvr'] - centos_version = helpers.filter_from_tag(git_tag, pkg_name) - - print(f'new version: {centos_version} | current version: {hs_version}') - if helpers.compare_versions(centos_version, hs_version) == 1: - issue = db.select_issue(pkg_name, conn) - hs_tag_name = db.select_tag(hs_tag_id, conn)[0] - new_issue = helpers.create_ticket(pkg_name, centos_version, hs_version, hs_tag_name, commit) - if issue: - db.update_issue_row(pkg_name, new_issue['id'], centos_version, conn) - is_open = helpers.is_issue_Open(centos_version) - if is_open: - helpers.comment_on_issue(issue[0], new_issue['id'], centos_version) - helpers.close_issue(issue[0]) - else: - db.insert_issue(new_issue['id'], pkg_name, centos_version, conn) - - -def listen_on_updates(conn): - """Sets up the mqtt client, starts the loop, and listens - on git.centos.org notifications for tag creation. - """ - - client = mqtt.Client() - client.on_connect = on_connect - client.on_message = on_message - client.tls_set(ca_certs=CAFILE, certfile=CERT, keyfile=KEY) - client.user_data_set({'db_conn': conn}) - client.connect(MQTT_BROKER, MQTT_PORT) - - client.loop_forever() - - -def main(): - print('running...') - try: - conn = db.get_connection() - setup(conn) - time.sleep(2) - listen_on_updates(conn) - except KeyboardInterrupt: - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/package_updates/settings/constants.py b/package_updates/settings/constants.py index 1350958..ab4568a 100644 --- a/package_updates/settings/constants.py +++ b/package_updates/settings/constants.py @@ -3,13 +3,18 @@ import random # URLS -CBS_URL = 'https://cbs.centos.org/koji/' -KOJIHUB_URL = 'https://kojihub.stream.centos.org/koji/' -PAGURE_REPO_API_URL = 'https://pagure.io/api/0/centos-sig-hyperscale/package-updates/' -PAGURE_REPO_URL = 'https://pagure.io/centos-sig-hyperscale/package-updates/' +CBS_URL = 'https://cbs.centos.org/kojihub' +CBS_BUILD_URL = 'https://cbs.centos.org/koji/buildinfo?buildID=' +KOJIHUB_URL = 'https://kojihub.stream.centos.org/kojihub' +PAGURE_REPO_API_URL = 'https://pagure.io/api/0/test-oidoming/' #'https://pagure.io/api/0/centos-sig-hyperscale/package-updates/' os.environ['PAGURE_REPO_API_URL'] repo api link where issues going to be created +PAGURE_REPO_URL = 'https://pagure.io/test-oidoming/' #'https://pagure.io/centos-sig-hyperscale/package-updates/' os.environ['PAGURE_REPO_URL'] GIT_CENTOS_API_URL = 'https://git.centos.org/api/0/rpms/' GIT_CENTOS_URL = 'https://git.centos.org/rpms/' + +# PAGURE API +PAGURE_API_KEY = os.environ['PAGURE_API_KEY'] + # Git branches C8S_GIT_BRANCH = 'c8s' @@ -21,7 +26,7 @@ C8S_GIT_BRANCH = 'c8s' """ MQTT_BROKER = 'mqtt.git.centos.org' MQTT_PORT = 8883 -TOPIC = 'git.centos.org/git.tag.creation' +MQTT_TOPIC = 'git.centos.org/git.tag.creation' CAFILE = os.environ['CAFILE'] CERT = os.environ['CERT'] KEY = os.environ['KEY'] diff --git a/package_updates/utils/db.py b/package_updates/utils/db.py index 3ec6d6c..43974e4 100755 --- a/package_updates/utils/db.py +++ b/package_updates/utils/db.py @@ -1,255 +1,286 @@ import sqlite3 +import re def get_connection() -> sqlite3.Connection: - """Gets an in-memory sqlite connection. + """Creates an in-memory sqlite connection. Returns: - A sqlite connection. + A sqlite connection with sqlite3.Row as the row factory. """ - return sqlite3.connect("file:mem1?mode=memory&cache=shared", uri=True) + conn = sqlite3.connect("file:mem1?mode=memory&cache=shared", uri=True, check_same_thread=False) + conn.row_factory = sqlite3.Row + return conn -def create_package_table(conn: sqlite3.Connection): - """Creates a sql table for hs packages. - - Args: - conn: sqlite3 connection - """ - - cursor = conn.cursor() - sql = """CREATE TABLE IF NOT EXISTS package ( - package_id INTEGER PRIMARY KEY, - name text, - version text, - release text, - distro_version, - build_id INTEGER, - tag_id INTEGER) - """ - - cursor.execute(sql) - conn.commit() - - -def insert_package(package_id: int, name: str, version: str, release: str, - build_id: int, tag_id: int, conn: sqlite3.Connection): - """Inserts a hs package to package table. - - Args: - package_id: CBS package id. - name: String of package name. - version: String of package version. - release: String of package release. - build_id: CBS package build id. - tag_id: CBS package tag id. - conn: sqlite3 connection. - """ - - cursor = conn.cursor() - cursor.execute("""INSERT INTO package( - package_id, - name, - version, - release, - build_id, - distro_version, - tag_id) - VALUES(?, ?, ?, ?, ?, ?, ?)""", - (package_id, name, version, release, build_id, tag_id)) - conn.commit() - - -def select_package(name: str, distro_version: str, conn: sqlite3.Connection) -> tuple: - """Gets a hs package from package table. - - Args: - name: String of package name. - conn: sqlite3 connection. - - Returns: - A tuple containing package_id, name, version, release, build_id, tag_id. +class BuildTargetDB: + """Contains sql operations/methods for buildTarget table. """ - cursor = conn.cursor() - cursor.execute("SELECT * FROM package WHERE name=? and distro_version=?", (name, distro_version)) - - return cursor.fetchone() - - -def select_packages(conn: sqlite3.Connection) -> list: - """Gets all hs packages from database. - - Args: - conn: sqlite3 connection. + def __init__(self, conn: sqlite3.Connection) -> None: + self.conn = conn - Returns: - A list of tuples of hs packages. - """ + def create_build_target_table(self) -> None: + cursor = self.conn.cursor() + sql = """CREATE TABLE IF NOT EXISTS buildTarget ( + build_target_id INTEGER PRIMARY KEY, + name text) + """ + + cursor.execute(sql) + self.conn.commit() - cursor = conn.cursor() - cursor.execute("SELECT * FROM package") - - return cursor.fetchall() - + def select_build_target(self, build_target_name: str) -> sqlite3.Row: + """Get a saved build target from buidTarget table. -def insert_packages_from_builds(packages: list, conn: sqlite3.Connection): - """Inserts a list of packages in the database. + Args: + build_target_name: String of cbs build target name. + + Returns: + A sqlite3.Row object containing build_target_id and name. + """ - Args: - package: list of package builds from cbs. - conn: sqlite3 connection. - """ - - for package in packages: - package_id = package['package_id'] - name = package['package_name'] - version = package['version'] - release = package['release'] - build_id = package['build_id'] - tag_id = package['tag_id'] - - result = select_package(name, conn) - if result: - continue + cursor = self.conn.cursor() + cursor.execute("SELECT * FROM buildTarget WHERE name=?", [build_target_name]) + return cursor.fetchone() - insert_package(package_id, name, version, release, build_id, tag_id, conn) + def insert_build_target(self, build_target_id: int, name: str) -> None: + """Inserts a (CBS) build target to buildTarget table. + Args: + build_target_id: build target id from cbs/koji + name: String of build target name from cbs/koji. + """ -def create_tag_table(conn: sqlite3.Connection): - """Creates the tag (from CBS) table. - - Args: - conn: sqlite3 connection. - """ + cursor = self.conn.cursor() + cursor.execute("""INSERT INTO buildTarget( + build_target_id, + name) + VALUES(?, ?)""", + (build_target_id, name)) + self.conn.commit() - cursor = conn.cursor() - sql = """CREATE TABLE IF NOT EXISTS tag ( - tag_id INTEGER PRIMARY KEY, - tag_name TEXT) - """ + def insert_build_targets(self, build_targets: list) -> None: + """Inserts a list of build targets into the database. - cursor.execute(sql) - conn.commit() + Args: + build_targets: list of dictionaries of build targets from cbs/koji. + """ + for build_target in build_targets: + self.insert_build_target(build_target['id'], build_target['name']) -def insert_tag(tag_id: int, tag_name: str, conn: sqlite3.Connection): - """Inserts a CBS tag to the tag table. - Args: - tag_id: CBS hs tag id. - tag_name: String of CBS hs tag name. - conn: sqlite3 connection. +class PackageDB: + """Contains sql operations/methods for package table. """ - cursor = conn.cursor() - cursor.execute("""INSERT INTO tag(tag_id, tag_name) VALUES(?, ?)""", (tag_id, tag_name)) - - conn.commit() - - -def insert_tags(tags: list, conn: sqlite3.Connection): - """Inserts a list of tags to the tag table. - - Args: - package: List of cbs tags. - conn: sqlite3 connection. - """ - - for tag in tags: - tag_id = tag[0] - tag_name = tag[1] - - result = select_tag(tag_id, conn) - if result: - continue - - insert_tag(tag_id, tag_name, conn) - - -def select_tag(tag_id: str, conn: sqlite3.Connection) -> tuple: - """Gets a CBS hs tag from tag table. + def __init__(self, conn: sqlite3.Connection) -> None: + self.conn = conn + + def create_package_table(self) -> None: + cursor = self.conn.cursor() + sql = """CREATE TABLE IF NOT EXISTS package ( + package_id INTEGER PRIMARY KEY, + name text, + version text, + release text, + distro_version text, + tag_id INTEGER, + FOREIGN KEY(tag_id) REFERENCES tag(tag_id)) + """ + + cursor.execute(sql) + self.conn.commit() + + def select_package(self, name: str, distro_version: str) -> sqlite3.Row: + """Gets a saved package from package table. + + Args: + name: String of package name. + distro_version: String of package distro. e.g. 'c9s'. + + Returns: + A sqlite3.Row object containing package_id, name, version, release, build_id, tag_id. + """ + + cursor = self.conn.cursor() + cursor.execute("SELECT * FROM package WHERE name=? and distro_version=?", (name, distro_version)) + return cursor.fetchone() + + def select_packages(self) -> list: + """Gets all saved packages from database. + + Returns: + A list of sqlite3.Rows containing saved packages. + """ + + cursor = self.conn.cursor() + cursor.execute("SELECT * FROM package") + return cursor.fetchall() + + def insert_package(self, name: str, version: str, release: str, + distro_version: str, tag_id: int) -> None: + """Inserts a package to package table. + + Args: + name: String of package name. + version: String of package version. + release: String of package release. + distro_version: package distro. e.g. 'c9s'. + tag_id: CBS/koji package tag id. + """ + + cursor = self.conn.cursor() + cursor.execute("""INSERT INTO package( + name, + version, + release, + distro_version, + tag_id) + VALUES(?, ?, ?, ?, ?)""", + (name, version, release, distro_version, tag_id)) + self.conn.commit() + + def insert_packages_from_builds(self, packages: list) -> None: + """Inserts a list of packages into the database. + + Args: + packages: list of package builds from cbs/koji. + """ + + for package in packages: + name = package['package_name'] + release = package['release'] + ext = re.search('el[0-9]+', release).group() # '1.hs.el8' -> el8 + distro_num = re.search('[0-9]+', ext).group() # el8 -> 8 + distro = f'c{distro_num}s' + + result = self.select_package(name, distro) + if result: + continue + + self.insert_package(name, package['version'], release, distro, package['tag_id']) + - Args: - tag_id: CBS hs tag id. - conn: sqlite3 connection. - - Returns: - A tuple containing the tag name. +class TagDB: + """Contains sql operations/methods for tag table. """ - cursor = conn.cursor() - cursor.execute("SELECT tag_name FROM tag WHERE tag_id=?", [tag_id]) + def __init__(self, conn: sqlite3.Connection) -> None: + self.conn = conn - return cursor.fetchone() + def create_tag_table(self): + cursor = self.conn.cursor() + sql = """CREATE TABLE IF NOT EXISTS tag ( + tag_id INTEGER PRIMARY KEY, + tag_name TEXT) + """ + cursor.execute(sql) + self.conn.commit() -def create_issue_table(conn: sqlite3.Connection): - """Creates a sql table for pagure issues. + def select_tag(self, tag_id: int) -> sqlite3.Row: + """Gets a tag from tag table. - Args: - conn: sqlite3 connection - """ + Args: + tag_id: tag id from cbs/koji. + + Returns: + A sqlite3.Row containing the tag name. + """ - cursor = conn.cursor() - sql = """CREATE TABLE IF NOT EXISTS issue ( - issue_id INTEGER PRIMARY KEY, - package_name text, - version_tag text) - """ + cursor = self.conn.cursor() + cursor.execute("SELECT tag_name FROM tag WHERE tag_id=?", [tag_id]) + return cursor.fetchone() - cursor.execute(sql) - conn.commit() + def insert_tag(self, tag_id: int, tag_name: str) -> None: + """Inserts a (CBS) tag into the tag table. + Args: + tag_id: tag id. + tag_name: String tag name. + """ -def insert_issue(issue_id: int, package_name: str, version_tag: str, conn: sqlite3.Connection): - """Inserts an issue to issue table. + cursor = self.conn.cursor() + cursor.execute("""INSERT INTO tag(tag_id, tag_name) VALUES(?, ?)""", (tag_id, tag_name)) + self.conn.commit() - Args: - package_name: String of package name. - issue_id: New issue id. - version_tag: String of new centos package version, e.g. dnf-4.7.0-8.el8. - conn: sqlite3 connection. - """ + def insert_tags(self, tags: list) -> None: + """Inserts a list of tags into the tag table. - cursor = conn.cursor() - cursor.execute("""INSERT INTO issue( - issue_id, - package_name, - version_tag) - VALUES(?, ?, ?)""", - (issue_id, package_name, version_tag)) - conn.commit() + Args: + tags: List of tuples of tags. + """ + for tag in tags: + tag_id = tag[0] + tag_name = tag[1] -def update_issue_row(package_name: str, issue_id: int, version_tag: str, conn: sqlite3.Connection): - """Updates an issue for a specific package in database. + result = self.select_tag(tag_id) + if result: + continue - Args: - package_name: String of package name. - issue_id: New issue id. - version_tag: String of new centos package version, e.g. dnf-4.7.0-8.el8. - conn: sqlite3 connection. - """ + self.insert_tag(tag_id, tag_name) - cursor = conn.cursor() - cursor.execute("UPDATE issue SET issue_id=?, version_tag=? WHERE package_name=?", (issue_id, version_tag, package_name)) - conn.commit() - -def select_issue(package_name: str, conn: sqlite3.Connection) -> tuple: - """Gets an issue from issue table. - - Args: - package_name: String of package name. - conn: sqlite3 connection. - - Returns: - A tuple containing issue_id, package_name, version_tag. +class IssueDB: + """Contains sql operations/methods for issue table. """ - cursor = conn.cursor() - cursor.execute("SELECT * FROM issue WHERE package_name=?", [package_name]) + def __init__(self, conn: sqlite3.Connection) -> None: + self.conn = conn + + def create_issue_table(self) -> None: + cursor = self.conn.cursor() + sql = """CREATE TABLE IF NOT EXISTS issue ( + issue_id INTEGER PRIMARY KEY, + package_id INTEGER, + FOREIGN KEY(package_id) REFERENCES package(package_id)) + """ + + cursor.execute(sql) + self.conn.commit() + + def select_issue(self, package_id: int) -> sqlite3.Row: + """Gets an issue from issue table. + + Args: + package_id: package id from the package table. + + Returns: + A sqlite3.Row object containing issue_id, tags, package_id. + """ + + cursor = self.conn.cursor() + cursor.execute("SELECT * FROM issue WHERE package_id=?", [package_id]) + return cursor.fetchone() - return cursor.fetchone() \ No newline at end of file + def insert_issue(self, issue_id: int, package_id: int) -> None: + """Inserts an issue to issue table. + + Args: + issue_id: Issue id from pagure.io. + package_id: package id from package table + """ + + cursor = self.conn.cursor() + cursor.execute("""INSERT INTO issue( + issue_id, + package_id) + VALUES(?, ?)""", + (issue_id, package_id)) + self.conn.commit() + + def update_issue_row(self, issue_id: int, package_id: int) -> None: + """Updates an issue for a specific package in database. + + Args: + issue_id: New issue id from pagure.io. + package_id: package id from package table + """ + + cursor = self.conn.cursor() + cursor.execute("UPDATE issue SET issue_id=? WHERE package_id=?", (issue_id, package_id)) + self.conn.commit() diff --git a/package_updates/utils/helpers.py b/package_updates/utils/helpers.py index 70fbea8..03cdc01 100755 --- a/package_updates/utils/helpers.py +++ b/package_updates/utils/helpers.py @@ -1,11 +1,11 @@ -import os +import re from urllib.parse import urljoin import rpm import hawkey import koji import requests -from ..settings.constants import GIT_CENTOS_API_URL, GIT_CENTOS_URL, PAGURE_REPO_API_URL, PAGURE_REPO_URL +from settings.constants import GIT_CENTOS_API_URL, GIT_CENTOS_URL, PAGURE_REPO_API_URL, PAGURE_API_KEY def get_koji_session(url: str) -> koji.ClientSession: @@ -26,7 +26,7 @@ def get_tagged_packages(session: koji.ClientSession, tag: int) -> list: Args: session: Koji client session. - tag: Tag ID. + tag: Tag ID of a koji/cbs tag. Returns: List of dictionaries with packages info. @@ -61,25 +61,20 @@ def join_tagged_packages(session: koji.ClientSession, tags: list) -> list: return packages -def get_latest_tagged_builds(session: koji.ClientSession, packages: list) -> list: +def get_latest_tagged_builds(session: koji.ClientSession, tags:list) -> list: """Gets the latest cbs build for a list of packages. Args: session: Koji client session. - packages: List of packages. + tags: List of koji/cbs tags. Returns: list of latest package builds. """ builds = [] - for package in packages: - package_name = package['package_name'] - build = session.listTagged(package['tag_id'], latest=True, package=package_name) - if len(build) == 0: - continue - builds.append(build[0]) - + for tag in tags: + builds += session.getLatestBuilds(tag) return builds @@ -103,6 +98,16 @@ def get_koji_tags(session: koji.ClientSession, tags_id: list) -> list: def get_koji_build_targets(session: koji.ClientSession, filter: str) -> list: + """Gets build targets from koji/cbs. + + Args: + session: Koji client session. + filter: keyword to filter build targets. e.g. 'hyperscale' will return all build targets that cointains this word. + + Returns: + list of build targets. + """ + targets = session.getBuildTargets() filtered_targets = [target for target in targets if filter in target['name']] return filtered_targets @@ -140,10 +145,9 @@ def split_package(pkg: str): subj = hawkey.Subject(pkg) nevra_possibilities = subj.get_nevra_possibilities() - epel = 'el8' for nevra in nevra_possibilities: - if epel in str(nevra.release): + if re.search('el[0-9]+', str(nevra.release)): return (str(nevra.name), str(nevra.version), str(nevra.release)) return None @@ -227,7 +231,28 @@ def get_git_pkg_version(package: str, branch: str) -> tuple: return (latest_version, commit) -def create_ticket(pkg: str, upstream_version: str, current_version: str, cbs_tag_name: str, commit: str): +def get_epel(release: str) -> str: + return re.search('el[0-9]+', release).group() + + +def get_centos_version(package: str) -> str: + """Gets centos distro version from package release + Returns string of centos version. e.g. 'c8s' + + Args: + package: String of package release. + + Returns: + String with centos version. + """ + #split_package(package) + ext = re.search('el[0-9]+', package).group() # '1.hs.el8' -> el8 + distro_num = re.search('[0-9]+', ext).group() # el8 -> 8 + distro = f'c{distro_num}s' + return distro + + +def create_ticket(pkg: str, upstream_version: str, current_version: str, cbs_tag_name: str, commit: str, distro: str): """Creates a ticket on paguire.io repo. Args: @@ -241,10 +266,10 @@ def create_ticket(pkg: str, upstream_version: str, current_version: str, cbs_tag 'issue_content': f"""Latest upstream release: {upstream_version} Current version/release: {current_version} URL: {commit_url}""", - 'tag': f'{cbs_tag_name},{upstream_version},{pkg}' + 'tag': f'{cbs_tag_name},{upstream_version},{pkg},{pkg}.{distro}' } url = urljoin(PAGURE_REPO_API_URL, 'new_issue') - token = os.environ['PAGURE_API_KEY'] + token = PAGURE_API_KEY headers = {'Authorization': f'access_token {token}'} print('Creating ticket on pagure.io...') @@ -328,7 +353,7 @@ def get_open_issue(tag: str) -> dict: return None -def comment_on_issue(prev_issue_id: int, new_issue_id: int, version_tag: str): +def comment_on_issue(issue_id: int, comment: str): """Adds a comment to a specific pagure.io issue. Args: @@ -337,14 +362,8 @@ def comment_on_issue(prev_issue_id: int, new_issue_id: int, version_tag: str): version_tag: String of new package version. """ - new_issue_url = urljoin(PAGURE_REPO_URL, f'issue/{new_issue_id}') - comment = { - 'comment': f"""{version_tag} is available and this issue still open. - New issue has been created for the newer version. - URL: {new_issue_url}""" - } - url = urljoin(PAGURE_REPO_API_URL, f'issue/{prev_issue_id}/comment') - token = os.environ['PAGURE_API_KEY'] + url = urljoin(PAGURE_REPO_API_URL, f'issue/{issue_id}/comment') + token = PAGURE_API_KEY headers = {'Authorization': f'access_token {token}'} try: @@ -360,7 +379,7 @@ def comment_on_issue(prev_issue_id: int, new_issue_id: int, version_tag: str): print(err) return - print(f'Comment added on issue {prev_issue_id}') + print(f'Comment added on issue {issue_id}') def get_issues(tags: list) -> list: @@ -390,19 +409,20 @@ def get_issues(tags: list) -> list: return issues -def close_issue(issue_id: int): +def close_issue(issue_id: int, close_status: str): """Closes a pagure.io issue. Args: issue_id: Issue id. + close_status: String of status (Fixed, Invalid, Duplicate, Insufficient Data) """ data = { 'status': 'Closed', - 'close_status': 'Invalid' + 'close_status': close_status } url = urljoin(PAGURE_REPO_API_URL, f'issue/{issue_id}/status') - token = os.environ['PAGURE_API_KEY'] + token = PAGURE_API_KEY headers = {'Authorization': f'access_token {token}'} try: @@ -415,28 +435,24 @@ def close_issue(issue_id: int): print('issue closed') -def issue_filter(package_name: str, issues: list) -> dict: - """Filters and returns an issue if has a tag with a +def issue_filter(issues: list, package_name: str) -> dict: + """Filters and returns an issue if it has a tag with a package name and a package version. Args: - package_name: String of package name. issues: List of pagure.io issues. + package_name: String of package name. Returns: dictionary containing the issue id, package name and version tag, None if there is no package name and package version. """ - - version_tag = '' + for issue in issues: - c = 0 for tag in issue['tags']: if package_name in tag: - c += 1 - if tag != package_name: + if split_package(tag) and get_epel(tag): version_tag = tag - if c == 2: - return {'issue_id': issue['id'], 'package_name': issue, 'version_tag': version_tag} + return {'issue_id': issue['id'], 'package_name': package_name, 'version_tag': version_tag} return None \ No newline at end of file diff --git a/tests/test_amqp_pub/publish.py b/tests/test_amqp_pub/publish.py new file mode 100644 index 0000000..f4fcd68 --- /dev/null +++ b/tests/test_amqp_pub/publish.py @@ -0,0 +1,27 @@ +from fedora_messaging import api, message, config + +config.conf.setup_logging() + +msg = message.Message(topic=u'nice.message', headers={u'niceness': u'very'}, + body={ + u'attribute': u"state", + u'build_id': 40526, + u'epoch': 1, + u'instance': 'primary', + u'name': u'dnf', + u'new': 1, + u'old': 0, + u'owner': u'kmodsbot', + u'release': u'14.1.hsx.el8', + u'request': [ + u'git+https://git.centos.org/rpms/dnf.git#30aec0245e1d1a4b2f982bd8fa719a80a01b1696', + u'hyperscale8s-packages-experimental-el8', + { + u'custom_user_metadata': {}, + u'wait_builds': [] + } + ], + u'task_id': 2940860, + u'version': u'4.7.0' + }) +api.publish(msg) \ No newline at end of file diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 33833b1..7ca71d1 100755 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -2,7 +2,8 @@ import unittest from package_updates.utils.helpers import ( get_latest_version_with_commit, compare_versions, - filter_from_tag + filter_from_tag, + get_epel ) @@ -94,5 +95,9 @@ class TestMain(unittest.TestCase): self.assertEqual(get_latest_version_with_commit('rpm', tags3, 'c8s'), ('rpm-4.14.3-22.el8', '63d1365d625c4a6bcf3eff7b5488f5554ad4ca44')) self.assertEqual(get_latest_version_with_commit('rpm', tags4, 'c8s'), ('rpm-4.14.3-21.el8', 'a9e8d80332d37e00d3e07237cd042dcf248991de')) - + def test_get_epel(self): + samples = ['libvirt-8.0.0-6.module+el8.7.0+15026+c30823f5', 'zlib-1.2.11-20.el8', 'util-linux-2.37.4-9.el9'] + epels = ['c8s', 'c8s', 'c9s'] + for sample, epel in zip(samples, epels): + self.assertEqual(get_epel(sample), epel) diff --git a/tests/test_mqtt_pub/publish.py b/tests/test_mqtt_pub/publish.py index 0ec4e46..36937b9 100755 --- a/tests/test_mqtt_pub/publish.py +++ b/tests/test_mqtt_pub/publish.py @@ -28,11 +28,15 @@ def publish(client): msg_count = 1 msg = b'' while True: - time.sleep(5) + time.sleep(30) if msg_count == 1: msg = b'{"repo": {"custom_keys": [], "name": "selinux-policy", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/selinux-policy", "url_path": "rpms/selinux-policy", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/selinux-policy-3.14.3-93.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' elif msg_count == 2: - msg = b'{"repo": {"custom_keys": [], "name": "dnf", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/dnf", "url_path": "rpms/dnf", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/dnf-4.7.0-9.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' + msg = b'{"repo": {"custom_keys": [], "name": "dnf", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/dnf", "url_path": "rpms/dnf", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/dnf-4.7.0-12.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' + elif msg_count == 3: + msg = b'{"repo": {"custom_keys": [], "name": "dnf", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/dnf", "url_path": "rpms/dnf", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/dnf-4.7.0-13.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' + elif msg_count == 4: + msg = b'{"repo": {"custom_keys": [], "name": "dnf", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/dnf", "url_path": "rpms/dnf", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/dnf-4.7.0-14.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' else: msg = b'{"repo": {"custom_keys": [], "name": "dnf", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/dnf", "url_path": "rpms/dnf", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/dnf-4.7.0-1.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' @@ -43,8 +47,8 @@ def publish(client): else: print(f"Failed to sent message to topic {topic}") - if msg_count == 3: - msg_count = 0 + if msg_count == 5: + break#msg_count = 0 msg_count += 1 print('----------------------------------------------') From 7b6e902548be7912477b64bc224619373e625c10 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sep 09 2022 17:53:48 +0000 Subject: [PATCH 5/7] Modify README.MD --- diff --git a/README.md b/README.md index 4f0d254..ad54f35 100755 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ ## Usage +MQTT ```bash git clone https://pagure.io/centos-sig-hyperscale/package-updates.git cd package-updates @@ -13,9 +14,28 @@ podman run -it \ -e CAFILE=/home/app/.centos-server-ca.cert \ -e CERT=/home/app/.centos.cert \ -e KEY=/home/app/.centos.cert \ + -e RUN_MODE=MQTT \ package-updates ``` +AMQP +```bash +git clone https://pagure.io/centos-sig-hyperscale/package-updates.git +cd package-updates +podman build -t package-updates . +podman run -it \ +--mount type=bind,src=$HOME/.centos-server-ca.cert,dst=/home/app/.centos-server-ca.cert,ro=true,relabel=private \ +--mount type=bind,src=$HOME/.centos.cert,dst=/home/app/.centos.cert,ro=true,relabel=private \ +--mount type=bind,source=fedora.toml,dst=/home/app/package-updates/fedora.toml +-e PAGURE_API_KEY= \ +-e CAFILE=/home/app/.centos-server-ca.cert \ +-e CERT=/home/app/.centos.cert \ +-e KEY=/home/app/.centos.cert \ +-e FEDORA_MESSAGING_CONF=/home/app/package-updates/fedora.toml \ +-e RUN_MODE=AMQP \ +package-updates +``` + ### Enviroment variables The script needs these enviroment variables: @@ -24,15 +44,20 @@ The script needs these enviroment variables: - CAFILE: .centos-server-ca.cert file - CERT: .centos.cert file - KEY: .centos.cert file +- FEDORA_MESSAGING_CONF: path to fedora messaging configuration file (fedora.toml) For more information on how to get the centos cert files, see: \ Cert files needed for MQTT, see the Message Broker (MQTT) section: +*Note: Inside fedora.toml file it is needed to specify the cacert.pem, fedora-key.pem and fedora-cert.pem files path. Files can be downloaded from here: https://github.com/fedora-infra/fedora-messaging/tree/stable/configs* + ### Test enviroment You can use tests/test_mqtt_pub/publish.py to run a mqtt publisher to test the script with a localhost mqtt server. -Note: git.centos.org notifications don't appear very often so if you want to test the git.centos.org server leave the script for a long time running (hours or days). +You can use tests/test_amqp_pub/publish.py to run a amqp publisher to test the script with the fedora messaging server. + +*Note: git.centos.org notifications don't appear very often so if you want to test the git.centos.org server leave the script for a long time running (hours or days).* ### MQTT payload To receive notifications from git.centos.org package updates the script listens to "git.centos.org/git.tag.creation" topic on mqtt, this topic is for git tag release creation. Here is an example of the payload for this topic: @@ -84,5 +109,38 @@ To receive notifications from git.centos.org package updates the script listens } ``` -## Script flow -![Script flow diagram](images/FlowDiagram.png) \ No newline at end of file +### AMQP payload +To receive notifications from cbs.centos.org builds the script listens to "org.centos.prod.cbs.buildsys.build.state.change" topic on amqp, this topic is for cbs/koji builds status change. Here is an example of the payload for this topic: +```json +{ + "body": { + "attribute": "state", + "build_id": 41063, + "epoch": null, + "instance": "primary", + "name": "zlib", + "new": 0, + "old": null, + "owner": "aekoroglu", + "release": "21.hs+intel.el8", + "request": [ + "git+https://git.centos.org/rpms/zlib.git#b65ea150e9ae1699e1227bee2dec884d4f2f1c0b", + "hyperscale8s-packages-intel-el8s", + { + "custom_user_metadata": {}, + "wait_builds": [] + } + ], + "task_id": 3000379, + "version": "1.2.11" + }, + "headers": { + "fedora_messaging_schema": "base.message", + "fedora_messaging_severity": 20, + "sent-at": "2022-09-08T15:58:07+00:00" + }, + "id": "cace298e-72dd-440f-a652-ddf8afcf1ca5", + "queue": null, + "topic": "org.centos.prod.cbs.buildsys.build.state.change" +} +``` \ No newline at end of file diff --git a/images/DiagramFlowAutoClose.png b/images/DiagramFlowAutoClose.png new file mode 100644 index 0000000..1d7bb52 Binary files /dev/null and b/images/DiagramFlowAutoClose.png differ diff --git a/package_updates/settings/constants.py b/package_updates/settings/constants.py index ab4568a..628a066 100644 --- a/package_updates/settings/constants.py +++ b/package_updates/settings/constants.py @@ -6,8 +6,8 @@ import random CBS_URL = 'https://cbs.centos.org/kojihub' CBS_BUILD_URL = 'https://cbs.centos.org/koji/buildinfo?buildID=' KOJIHUB_URL = 'https://kojihub.stream.centos.org/kojihub' -PAGURE_REPO_API_URL = 'https://pagure.io/api/0/test-oidoming/' #'https://pagure.io/api/0/centos-sig-hyperscale/package-updates/' os.environ['PAGURE_REPO_API_URL'] repo api link where issues going to be created -PAGURE_REPO_URL = 'https://pagure.io/test-oidoming/' #'https://pagure.io/centos-sig-hyperscale/package-updates/' os.environ['PAGURE_REPO_URL'] +PAGURE_REPO_API_URL = 'https://pagure.io/api/0/centos-sig-hyperscale/package-updates/' +PAGURE_REPO_URL = 'https://pagure.io/centos-sig-hyperscale/package-updates/' GIT_CENTOS_API_URL = 'https://git.centos.org/api/0/rpms/' GIT_CENTOS_URL = 'https://git.centos.org/rpms/' From 81a858112adf9379a745d00ceae127142f952683 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sep 21 2022 18:28:50 +0000 Subject: [PATCH 6/7] Fix dockerfile user permissions & docstrings --- diff --git a/Dockerfile b/Dockerfile index 2fcaa8c..4b80a38 100755 --- a/Dockerfile +++ b/Dockerfile @@ -10,22 +10,21 @@ RUN dnf update -y && \ RUN dnf clean all -ENV HOME /home/app -RUN useradd -r -d $HOME app -RUN usermod -aG wheel app -WORKDIR /home/app/package-updates +RUN mkdir -p /app/package-updates +WORKDIR /app/package-updates -RUN chown -R app:wheel $HOME/package-updates +ADD --chown=1001:0 https://raw.githubusercontent.com/fedora-infra/fedora-messaging/stable/configs/fedora-key.pem /app/package-updates +ADD --chown=1001:0 https://raw.githubusercontent.com/fedora-infra/fedora-messaging/stable/configs/fedora-cert.pem /app/package-updates +ADD --chown=1001:0 https://raw.githubusercontent.com/fedora-infra/fedora-messaging/stable/configs/cacert.pem /app/package-updates -ADD --chown=app:wheel https://raw.githubusercontent.com/fedora-infra/fedora-messaging/stable/configs/fedora-key.pem $HOME/package-updates/ -ADD --chown=app:wheel https://raw.githubusercontent.com/fedora-infra/fedora-messaging/stable/configs/fedora-cert.pem $HOME/package-updates/ -ADD --chown=app:wheel https://raw.githubusercontent.com/fedora-infra/fedora-messaging/stable/configs/cacert.pem $HOME/package-updates/ +COPY --chown=1001:0 ./package_updates/ /app/package-updates +COPY --chown=1001:0 ./fedora.toml /app/package-updates +COPY --chown=1001:0 ./requirements.txt /app/package-updates -COPY --chown=app:wheel ./package_updates/ $HOME/package-updates/ -COPY --chown=app:wheel ./fedora.toml $HOME/package-updates/ -COPY --chown=app:wheel ./requirements.txt $HOME/package-updates/ +RUN chgrp -R 0 /app && \ + chmod -R g=u /app RUN pip3 install -r requirements.txt ENTRYPOINT ["python3", "./main.py"] -USER app +USER 1001 diff --git a/fedora.toml b/fedora.toml index 9e381f1..75e5694 100644 --- a/fedora.toml +++ b/fedora.toml @@ -6,9 +6,9 @@ amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub" callback = "fedora_messaging.example:printer" [tls] -ca_cert = "/home/app/package-updates/cacert.pem" -keyfile = "/home/app/package-updates/fedora-key.pem" -certfile = "/home/app/package-updates/fedora-cert.pem" +ca_cert = "/app/package-updates/cacert.pem" +keyfile = "/app/package-updates/fedora-key.pem" +certfile = "/app/package-updates/fedora-cert.pem" [client_properties] app = "Example Application" diff --git a/package_updates/amqp/amqp.py b/package_updates/amqp/amqp.py index 1589af9..3984110 100644 --- a/package_updates/amqp/amqp.py +++ b/package_updates/amqp/amqp.py @@ -22,6 +22,7 @@ class AMQP: def __issue_closer(self, build: PackageBuild) -> None: """Auto close an issue for a specific package if it's open. """ + centos_version = helpers.get_centos_version(build.release) issue = helpers.get_open_issue(f'{build.name}.{centos_version}') if issue: @@ -53,7 +54,6 @@ class AMQP: def listen_on_amqp(self): """Connects to fedora messaging AMQP broker and keeps listening """ + api.twisted_consume(self.__on_message) reactor.run() - - diff --git a/package_updates/mqtt/mqtt.py b/package_updates/mqtt/mqtt.py index 1823d32..82be156 100644 --- a/package_updates/mqtt/mqtt.py +++ b/package_updates/mqtt/mqtt.py @@ -67,7 +67,6 @@ class MQTT: is_open = helpers.is_issue_Open(f'{pkg_name}.c8s') new_issue = helpers.create_ticket(pkg_name, centos_version, hs_version, hs_tag_name, package_release.commit, C8S_GIT_BRANCH) if issue: - print('issueee1') issue_db.update_issue_row(new_issue['id'], saved_hs_package['package_id']) if is_open: new_issue_url = urljoin(PAGURE_REPO_URL, f"issue/{new_issue['id']}") @@ -79,7 +78,6 @@ class MQTT: helpers.comment_on_issue(issue['issue_id'], comment) helpers.close_issue(issue['issue_id'], 'Invalid') else: - print('no issuee') issue_db.insert_issue(new_issue['id'], saved_hs_package['package_id']) def listen_on_updates(self): diff --git a/package_updates/settings/constants.py b/package_updates/settings/constants.py index 628a066..b31f541 100644 --- a/package_updates/settings/constants.py +++ b/package_updates/settings/constants.py @@ -31,7 +31,6 @@ CAFILE = os.environ['CAFILE'] CERT = os.environ['CERT'] KEY = os.environ['KEY'] client_id = f'hyperscale-sig-{random.randint(0, 1000)}' -RUN_MODE = os.getenv('RUN_MODE', default='MQTT') # AMQP @@ -50,4 +49,4 @@ HS_CBS_TAGS = [H8S_MAIN_TAG_ID, H8S_HOTFIXES_TAG_ID, H8S_EXP_TAG_ID, # Run mode -RUN_MODE = os.getenv('RUN_MODE', 'MQTT') \ No newline at end of file +RUN_MODE = os.getenv('RUN_MODE', default='MQTT') \ No newline at end of file diff --git a/package_updates/utils/helpers.py b/package_updates/utils/helpers.py index 03cdc01..016e341 100755 --- a/package_updates/utils/helpers.py +++ b/package_updates/utils/helpers.py @@ -172,6 +172,7 @@ def get_latest_version_with_commit(pkg: str, tags: list, branch: str) -> tuple: """Filters latest git tag of a package. Args: + pkg: String of the name. tags: List of git tags. branch: String of git branch. @@ -245,7 +246,7 @@ def get_centos_version(package: str) -> str: Returns: String with centos version. """ - #split_package(package) + ext = re.search('el[0-9]+', package).group() # '1.hs.el8' -> el8 distro_num = re.search('[0-9]+', ext).group() # el8 -> 8 distro = f'c{distro_num}s' @@ -256,8 +257,12 @@ def create_ticket(pkg: str, upstream_version: str, current_version: str, cbs_tag """Creates a ticket on paguire.io repo. Args: - pkg: String of package name. - tag_version: String of package version. + pkg: Package name. + upstream_version: New centos upstream version. + current_version: Current package version. + cbs_tag_name: Tag name from package. + commit: Commit id from source. + distro: Centos version """ commit_url = urljoin(GIT_CENTOS_URL, f'{pkg}/tree/{commit}') @@ -353,13 +358,12 @@ def get_open_issue(tag: str) -> dict: return None -def comment_on_issue(issue_id: int, comment: str): +def comment_on_issue(issue_id: int, comment: dict): """Adds a comment to a specific pagure.io issue. Args: - prev_issue_id: Issue id of the issue to add a comment, - new_issue_id: Issue Id of the new created issue, lo leave a link of the new issue. - version_tag: String of new package version. + issue_id: Issue id of the issue to add a comment, + comment: Comment body. """ url = urljoin(PAGURE_REPO_API_URL, f'issue/{issue_id}/comment') From 4dd1200be0f9fd490f3139d62a152f5b0bd134db Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Oct 24 2022 20:17:58 +0000 Subject: [PATCH 7/7] Change unclear documentation/code --- diff --git a/package_updates/amqp/amqp.py b/package_updates/amqp/amqp.py index 3984110..3da01af 100644 --- a/package_updates/amqp/amqp.py +++ b/package_updates/amqp/amqp.py @@ -36,7 +36,7 @@ class AMQP: helpers.close_issue(issue['id'], 'Fixed') def __on_message(self, msg): - """Callback. Called when a message is resieved by fedora messaging, checks if a + """Callback. Called when a message is received by fedora messaging, checks if a package build from cbs is complete and if the build target is saved in the DB. """ diff --git a/package_updates/amqp/package_build.py b/package_updates/amqp/package_build.py index a0172de..f155e6f 100644 --- a/package_updates/amqp/package_build.py +++ b/package_updates/amqp/package_build.py @@ -22,9 +22,7 @@ class PackageBuild: """Checks if a package build from koji is complete. """ - if self.build_status == 1: - return True - return False + return self.build_status == 1 def get_build_url(self) -> str: """Generates a url for a koji package build. diff --git a/package_updates/utils/helpers.py b/package_updates/utils/helpers.py index 016e341..21ddf28 100755 --- a/package_updates/utils/helpers.py +++ b/package_updates/utils/helpers.py @@ -327,14 +327,13 @@ def is_issue_Open(tag: str) -> bool: def get_open_issue(tag: str) -> dict: - """Verify if the issue for a specific tag is open. + """Gets an open issue containing a specific issue tag. Args: tag: String of issue tag. Returns: - True if the issue for that package update - has been created, false otherwise. + A dictionary with the issue info if there is an open issue, None otherwise. """ params = {