From 15e338318ccd82b50c782a9f1748bdca8bd2ea29 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Nov 10 2022 00:32:38 +0000 Subject: Added config option --- diff --git a/README.md b/README.md index ad54f35..e48727b 100755 --- a/README.md +++ b/README.md @@ -10,10 +10,12 @@ podman build -t package-updates . podman run -it \ --mount type=bind,src=$HOME/.centos-server-ca.cert,dst=/home/app/.centos-server-ca.cert,ro=true,relabel=private \ --mount type=bind,src=$HOME/.centos.cert,dst=/home/app/.centos.cert,ro=true,relabel=private \ + --mount type=bind,source=config.json,dst=/home/app/package-updates/config.json \ -e PAGURE_API_KEY= \ -e CAFILE=/home/app/.centos-server-ca.cert \ -e CERT=/home/app/.centos.cert \ -e KEY=/home/app/.centos.cert \ + -e CONFIG_PATH=/home/app/package-updates/config.json \ -e RUN_MODE=MQTT \ package-updates ``` @@ -26,16 +28,53 @@ podman build -t package-updates . podman run -it \ --mount type=bind,src=$HOME/.centos-server-ca.cert,dst=/home/app/.centos-server-ca.cert,ro=true,relabel=private \ --mount type=bind,src=$HOME/.centos.cert,dst=/home/app/.centos.cert,ro=true,relabel=private \ ---mount type=bind,source=fedora.toml,dst=/home/app/package-updates/fedora.toml +--mount type=bind,source=config.json,dst=/home/app/package-updates/config.json \ -e PAGURE_API_KEY= \ -e CAFILE=/home/app/.centos-server-ca.cert \ -e CERT=/home/app/.centos.cert \ -e KEY=/home/app/.centos.cert \ +-e CONFIG_PATH=/home/app/package-updates/config.json \ -e FEDORA_MESSAGING_CONF=/home/app/package-updates/fedora.toml \ -e RUN_MODE=AMQP \ package-updates ``` +### Configuration + +Use config.json to specify your own project configuration values. +Here you can specify: +- MQTT: + - The MQTT broker, port, and topic. +- Pagure: + - repo_url: Your desired Pagure repo URL, here is where the package updates tickets are going to be created. + - api_repo_url: Pagure repo API endpoint for repo_url. +- Cbs: + - tags: Koji CBS tags. Add cbs tag ids if you want to get package updates from a specific list of cbs tags. + - tags_keyword. If this is specified, you will get package updates from all cbs tags that contains this keyword. + +Config.json example +```json +{ + "mqtt": { + "broker": "mqtt.git.centos.org", + "port": 8883, + "topic": "git.centos.org/git.tag.creation" + }, + "pagure": { + "repo_url": "https://pagure.io/centos-sig-hyperscale/package-updates/", + "api_repo_url": "https://pagure.io/api/0/centos-sig-hyperscale/package-updates/" + }, + "cbs": { + "tags" : [], + "tags_keyword": "hyperscale" + } +} +``` + +*The tags filed is prioritized over the keyword field, if you provide both you will only get package updates from the list of tags specified.* + +*For AMQP configuration use the fedora.toml file* + ### Enviroment variables The script needs these enviroment variables: @@ -44,6 +83,7 @@ The script needs these enviroment variables: - CAFILE: .centos-server-ca.cert file - CERT: .centos.cert file - KEY: .centos.cert file +- CONFIG_PATH: path to config.json file - FEDORA_MESSAGING_CONF: path to fedora messaging configuration file (fedora.toml) For more information on how to get the centos cert files, see: \ diff --git a/config.json b/config.json new file mode 100644 index 0000000..23094a5 --- /dev/null +++ b/config.json @@ -0,0 +1,15 @@ +{ + "mqtt": { + "broker": "mqtt.git.centos.org", + "port": 8883, + "topic": "git.centos.org/git.tag.creation" + }, + "pagure": { + "repo_url": "", + "api_repo_url": "" + }, + "cbs": { + "tags" : [], + "tags_keyword": "" + } +} \ No newline at end of file diff --git a/package_updates/main.py b/package_updates/main.py index 43d4056..7fa0bc8 100644 --- a/package_updates/main.py +++ b/package_updates/main.py @@ -7,7 +7,7 @@ from urllib.parse import urljoin from amqp.amqp import AMQP from mqtt.mqtt import MQTT from utils import helpers, db -from settings.constants import HS_CBS_TAGS, CBS_URL, C8S_GIT_BRANCH, RUN_MODE, PAGURE_REPO_URL +from settings.constants import CBS_URL, CBS_TAGS, CBS_TAGS_KEYWORD, C8S_GIT_BRANCH, RUN_MODE, PAGURE_REPO_URL def setup(conn): @@ -15,16 +15,19 @@ def setup(conn): update is available, for first time running. """ - tags = HS_CBS_TAGS - session = helpers.get_koji_session(CBS_URL) - cbs_tags = helpers.get_koji_tags(session, tags) + + if len(CBS_TAGS) > 0: + tags = helpers.get_koji_tags_info(session, CBS_TAGS) + else: + tags = helpers.get_koji_tags(session, CBS_TAGS_KEYWORD) + builds = helpers.get_latest_tagged_builds(session, tags) - targets = helpers.get_koji_build_targets(session, 'hyperscale') #gets all hyperscale cbs build targets + targets = helpers.get_koji_build_targets(session, CBS_TAGS_KEYWORD) tag_db = db.TagDB(conn) tag_db.create_tag_table() - tag_db.insert_tags(cbs_tags) + tag_db.insert_tags(tags) package_db = db.PackageDB(conn) package_db.create_package_table() diff --git a/package_updates/settings/constants.py b/package_updates/settings/constants.py index b31f541..ce1e731 100644 --- a/package_updates/settings/constants.py +++ b/package_updates/settings/constants.py @@ -1,15 +1,20 @@ +import json import os import random +CONFIG_PATH = os.getenv('CONFIG_PATH') #path to config.json +with open(CONFIG_PATH) as f: + config = json.load(f) + # URLS CBS_URL = 'https://cbs.centos.org/kojihub' CBS_BUILD_URL = 'https://cbs.centos.org/koji/buildinfo?buildID=' KOJIHUB_URL = 'https://kojihub.stream.centos.org/kojihub' -PAGURE_REPO_API_URL = 'https://pagure.io/api/0/centos-sig-hyperscale/package-updates/' -PAGURE_REPO_URL = 'https://pagure.io/centos-sig-hyperscale/package-updates/' GIT_CENTOS_API_URL = 'https://git.centos.org/api/0/rpms/' GIT_CENTOS_URL = 'https://git.centos.org/rpms/' +PAGURE_REPO_URL = config['pagure']['repo_url'] or 'https://localhost/foo/bar/' +PAGURE_REPO_API_URL = config['pagure']['api_repo_url'] or 'https://localhost/api/0/foo/bar/' # PAGURE API @@ -24,9 +29,9 @@ C8S_GIT_BRANCH = 'c8s' """ See https://wiki.centos.org/Sources for more details for the following MQTT values. """ -MQTT_BROKER = 'mqtt.git.centos.org' -MQTT_PORT = 8883 -MQTT_TOPIC = 'git.centos.org/git.tag.creation' +MQTT_BROKER = config['mqtt']['broker'] or 'mqtt.git.centos.org' +MQTT_PORT = config['mqtt']['port'] or 8883 +MQTT_TOPIC = config['mqtt']['topic'] or 'git.centos.org/git.tag.creation' CAFILE = os.environ['CAFILE'] CERT = os.environ['CERT'] KEY = os.environ['KEY'] @@ -38,14 +43,8 @@ CBS_BUILD_TOPIC = 'org.centos.prod.cbs.buildsys.build.state.change' # CBS tags -H8S_MAIN_TAG_ID = 2249 # hyperscale8s-packages-main-release cbs koji tag -H8S_HOTFIXES_TAG_ID = 2305 # hyperscale8s-packages-hotfixes-release cbs koji tag -H8S_EXP_TAG_ID = 2245 # hyperscale8s-packages-experimental-release cbs koji tag -H8S_INTEL_CANDIDATE_TAG_ID = 2614 # hyperscale8s-packages-intel-candidate -H8S_INTEL_TESTING_TAG_ID = 2615 # hyperscale8s-packages-intel-testing -H8S_INTEL_RELEASE_TAG_ID = 2616 # hyperscale8s-packages-intel-release -HS_CBS_TAGS = [H8S_MAIN_TAG_ID, H8S_HOTFIXES_TAG_ID, H8S_EXP_TAG_ID, - H8S_INTEL_CANDIDATE_TAG_ID, H8S_INTEL_TESTING_TAG_ID, H8S_INTEL_RELEASE_TAG_ID] +CBS_TAGS = config['cbs']['tags'] +CBS_TAGS_KEYWORD = config['cbs']['tags_keyword'] # Run mode diff --git a/package_updates/utils/db.py b/package_updates/utils/db.py index 43974e4..fbff923 100755 --- a/package_updates/utils/db.py +++ b/package_updates/utils/db.py @@ -215,8 +215,8 @@ class TagDB: """ for tag in tags: - tag_id = tag[0] - tag_name = tag[1] + tag_id = tag['id'] + tag_name = tag['name'] result = self.select_tag(tag_id) if result: diff --git a/package_updates/utils/helpers.py b/package_updates/utils/helpers.py index 21ddf28..ccdcba2 100755 --- a/package_updates/utils/helpers.py +++ b/package_updates/utils/helpers.py @@ -74,11 +74,11 @@ def get_latest_tagged_builds(session: koji.ClientSession, tags:list) -> list: builds = [] for tag in tags: - builds += session.getLatestBuilds(tag) + builds += session.getLatestBuilds(tag['id']) return builds -def get_koji_tags(session: koji.ClientSession, tags_id: list) -> list: +def get_koji_tags_info(session: koji.ClientSession, tag_ids: list) -> list: """Gets the tag id and tag name from koji cbs. Args: @@ -90,13 +90,34 @@ def get_koji_tags(session: koji.ClientSession, tags_id: list) -> list: """ tags = [] - for tag_id in tags_id: + for tag_id in tag_ids: tag = session.getTag(tag_id) - tags.append((tag['id'], tag['name'])) + tags.append(tag) return tags +def get_koji_tags(session: koji.ClientSession, keyword: str = None) -> list: + """Gets a list of dictionary tags from koji. + + Args: + session: Koji client session. + keywords: String of filter for tags. E.g. If the word "hyperscale" + is provided it will return all tags containg this word. + + Returns: + list of dictionaries containg koji tags. + """ + + koji_tags = session.listTags() + + if keyword: + tags = [tag for tag in koji_tags if keyword in tag['name']] + return tags + + return koji_tags + + def get_koji_build_targets(session: koji.ClientSession, filter: str) -> list: """Gets build targets from koji/cbs.