From bb9d86db8b30ceb035fdbcf73a9103e8a62842bb Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Jul 13 2023 13:19:14 +0000
Subject: [PATCH 1/4] create initial repo for sidetag


Related: https://pagure.io/koji/issue/3808

---

diff --git a/plugins/hub/sidetag_hub.py b/plugins/hub/sidetag_hub.py
index 85c12af..9b5e824 100644
--- a/plugins/hub/sidetag_hub.py
+++ b/plugins/hub/sidetag_hub.py
@@ -2,6 +2,10 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
+import json
+import logging
+import os
+import shutil
 import koji
 from koji.context import context
 from koji.plugin import callback, export
@@ -20,9 +24,16 @@ from kojihub import (
     get_user,
     policy_get_user,
     readInheritanceData,
+    repo_init,
+    repo_problem,
+    repo_ready,
+    RootExports,
 )
 from kojihub.db import QueryProcessor, nextval
 
+rootexports = RootExports()
+
+logger = logging.getLogger('koji.plugin.sidetag')
 
 CONFIG_FILE = "/etc/koji-hub/plugins/sidetag.conf"
 CONFIG = None
@@ -87,6 +98,7 @@ def createSideTag(basetag, debuginfo=False, suffix=None):
     :returns dict: sidetag name + id
     """
 
+    logger.debug(f'Creating sidetag for {basetag}')
     if suffix and suffix not in ALLOWED_SUFFIXES:
         raise koji.GenericError("%s suffix is not allowed for sidetag" % suffix)
 
@@ -136,7 +148,70 @@ def createSideTag(basetag, debuginfo=False, suffix=None):
     )
     _create_build_target(sidetag_name, sidetag_id, sidetag_id)
 
-    return {"name": sidetag_name, "id": sidetag_id}
+    # Sidetag is ready now, we try to create initial repo but if it is not successful
+    # kojira will take proper care later
+    return_data = {"name": sidetag_name, "id": sidetag_id}
+
+    # In case it it is not maven repo, we can copy old one and don't wait for kojira
+    # for maven and/or src, leave it on kojira
+    if basetag.get('maven_support'):
+        # maven repos contains too many files to be handled sync in hub call, let kojira do that
+        logger.debug(f'Not creating initial repo for {sidetag_name} due to maven_support')
+        return return_data
+
+    # find active basetag repo
+    repo_info = rootexports.getRepo(basetag, state=koji.REPO_READY)
+    if not repo_info:
+        return return_data
+
+    src_repo = koji.pathinfo.repo(repo_info['id'], basetag['name'])
+    try:
+        repo_json = json.load(open(os.path.join(src_repo, 'repo.json')))
+    except IOError:
+        logger.debug("Can't open repo.json for repo {repo_info['id']}")
+        return return_data
+
+    if debuginfo and not repo_json['with_debuginfo']:
+        logger.debug(f'Not creating initial repo for {sidetag_name}'
+                     ' as it requires debuginfo and source not')
+
+    sidetag = get_tag(sidetag_id, strict=True)
+
+    # TODO: without event it could be already irrelevant (even if not expired yet)
+    # maybe at least checking tag_changed_since_event? It would consume some CPU and
+    # user is maybe not interested in most actual repo (if yes, kojira will still
+    # regenerate it later)
+
+    logger.debug(f'Copying repo {repo_info["id"]} for sidetag')
+    new_repo_id, event_id = repo_init(sidetag['name'], with_debuginfo=debuginfo)
+
+    try:
+        # copy repodata
+        # TODO: Does it make sense to run RepoDone callbacks? Probably not as it is hard copy.
+        # koji.plugin.run_callbacks('preRepoDone', repo=repo_info, data=data, expire=False)
+        dst_repo = koji.pathinfo.repo(new_repo_id, sidetag['name'])
+        arches = [koji.canonArch(a) for a in koji.parse_arches(sidetag['arches'], to_list=True)]
+        if repo_json['with_separate_src']:
+            arches.append('src')
+        for arch in arches:
+            src_repodata = f'{src_repo}/{arch}/repodata'
+            dst_repodata = f'{dst_repo}/{arch}/repodata'
+            logger.debug(f'Copying repodata {src_repodata} to {dst_repodata}')
+            if os.path.exists(src_repodata):
+                # TODO: Could be hard linking dangerous here?
+                shutil.copytree(src_repodata, dst_repodata, copy_function=os.link)
+        # mimick host.repoDone, we're sure that there is no previous repo,
+        # so we can omit most steps
+        latestrepolink = koji.pathinfo.repo('latest', sidetag['name'])
+        os.symlink(str(new_repo_id), latestrepolink)
+        repo_ready(new_repo_id)
+        # koji.plugin.run_callbacks('postRepoDone', repo=repo_info, data=data, expire=False)
+    except Exception:
+        logger.error(f"Copying repo {repo_info['id']} to {new_repo_id} failed.")
+        repo_problem(new_repo_id)
+        raise
+
+    return return_data
 
 
 @export

From fdedf0936a7336063f6523d3824141d02eed6061 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Jul 13 2023 13:19:14 +0000
Subject: [PATCH 2/4] trigger newRepo task for new sidetag


Related: https://pagure.io/koji/issue/3808

---

diff --git a/plugins/cli/sidetag_cli.py b/plugins/cli/sidetag_cli.py
index 58137c5..a8e16f8 100644
--- a/plugins/cli/sidetag_cli.py
+++ b/plugins/cli/sidetag_cli.py
@@ -9,8 +9,7 @@ from optparse import OptionParser
 
 import koji
 from koji.plugin import export_cli
-from koji_cli.commands import anon_handle_wait_repo
-from koji_cli.lib import activate_session, arg_filter
+from koji_cli.lib import activate_session, arg_filter, watch_tasks
 
 
 @export_cli
@@ -38,7 +37,7 @@ def handle_add_sidetag(options, session, args):
     if opts.suffix:
         kwargs['suffix'] = opts.suffix
     try:
-        tag = session.createSideTag(basetag, **kwargs)
+        info = session.createSideTag(basetag, **kwargs)
     except koji.ActionNotAllowed:
         parser.error("Policy violation")
     except koji.ParameterError as ex:
@@ -48,13 +47,11 @@ def handle_add_sidetag(options, session, args):
             raise
 
     if not opts.quiet:
-        print(tag["name"])
+        print(info["name"])
 
     if opts.wait:
-        args = ["--target", tag["name"]]
-        if opts.quiet:
-            args.append("--quiet")
-        anon_handle_wait_repo(options, session, args)
+        return watch_tasks(session, [info['task_id']], quiet=opts.quiet,
+                           poll_interval=options.poll_interval, topurl=options.topurl)
 
 
 @export_cli
diff --git a/plugins/hub/sidetag_hub.py b/plugins/hub/sidetag_hub.py
index 9b5e824..4ff1576 100644
--- a/plugins/hub/sidetag_hub.py
+++ b/plugins/hub/sidetag_hub.py
@@ -2,10 +2,6 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import json
-import logging
-import os
-import shutil
 import koji
 from koji.context import context
 from koji.plugin import callback, export
@@ -22,19 +18,12 @@ from kojihub import (
     get_build_target,
     get_tag,
     get_user,
+    make_task,
     policy_get_user,
     readInheritanceData,
-    repo_init,
-    repo_problem,
-    repo_ready,
-    RootExports,
 )
 from kojihub.db import QueryProcessor, nextval
 
-rootexports = RootExports()
-
-logger = logging.getLogger('koji.plugin.sidetag')
-
 CONFIG_FILE = "/etc/koji-hub/plugins/sidetag.conf"
 CONFIG = None
 ALLOWED_SUFFIXES = []
@@ -98,7 +87,6 @@ def createSideTag(basetag, debuginfo=False, suffix=None):
     :returns dict: sidetag name + id
     """
 
-    logger.debug(f'Creating sidetag for {basetag}')
     if suffix and suffix not in ALLOWED_SUFFIXES:
         raise koji.GenericError("%s suffix is not allowed for sidetag" % suffix)
 
@@ -148,70 +136,11 @@ def createSideTag(basetag, debuginfo=False, suffix=None):
     )
     _create_build_target(sidetag_name, sidetag_id, sidetag_id)
 
-    # Sidetag is ready now, we try to create initial repo but if it is not successful
-    # kojira will take proper care later
-    return_data = {"name": sidetag_name, "id": sidetag_id}
-
-    # In case it it is not maven repo, we can copy old one and don't wait for kojira
-    # for maven and/or src, leave it on kojira
-    if basetag.get('maven_support'):
-        # maven repos contains too many files to be handled sync in hub call, let kojira do that
-        logger.debug(f'Not creating initial repo for {sidetag_name} due to maven_support')
-        return return_data
-
-    # find active basetag repo
-    repo_info = rootexports.getRepo(basetag, state=koji.REPO_READY)
-    if not repo_info:
-        return return_data
-
-    src_repo = koji.pathinfo.repo(repo_info['id'], basetag['name'])
-    try:
-        repo_json = json.load(open(os.path.join(src_repo, 'repo.json')))
-    except IOError:
-        logger.debug("Can't open repo.json for repo {repo_info['id']}")
-        return return_data
+    # little higher priority than other newRepo tasks
+    args = koji.encode_args(sidetag_name, debuginfo=debuginfo)
+    task_id = make_task('newRepo', args, priority=14, channel='createrepo')
 
-    if debuginfo and not repo_json['with_debuginfo']:
-        logger.debug(f'Not creating initial repo for {sidetag_name}'
-                     ' as it requires debuginfo and source not')
-
-    sidetag = get_tag(sidetag_id, strict=True)
-
-    # TODO: without event it could be already irrelevant (even if not expired yet)
-    # maybe at least checking tag_changed_since_event? It would consume some CPU and
-    # user is maybe not interested in most actual repo (if yes, kojira will still
-    # regenerate it later)
-
-    logger.debug(f'Copying repo {repo_info["id"]} for sidetag')
-    new_repo_id, event_id = repo_init(sidetag['name'], with_debuginfo=debuginfo)
-
-    try:
-        # copy repodata
-        # TODO: Does it make sense to run RepoDone callbacks? Probably not as it is hard copy.
-        # koji.plugin.run_callbacks('preRepoDone', repo=repo_info, data=data, expire=False)
-        dst_repo = koji.pathinfo.repo(new_repo_id, sidetag['name'])
-        arches = [koji.canonArch(a) for a in koji.parse_arches(sidetag['arches'], to_list=True)]
-        if repo_json['with_separate_src']:
-            arches.append('src')
-        for arch in arches:
-            src_repodata = f'{src_repo}/{arch}/repodata'
-            dst_repodata = f'{dst_repo}/{arch}/repodata'
-            logger.debug(f'Copying repodata {src_repodata} to {dst_repodata}')
-            if os.path.exists(src_repodata):
-                # TODO: Could be hard linking dangerous here?
-                shutil.copytree(src_repodata, dst_repodata, copy_function=os.link)
-        # mimick host.repoDone, we're sure that there is no previous repo,
-        # so we can omit most steps
-        latestrepolink = koji.pathinfo.repo('latest', sidetag['name'])
-        os.symlink(str(new_repo_id), latestrepolink)
-        repo_ready(new_repo_id)
-        # koji.plugin.run_callbacks('postRepoDone', repo=repo_info, data=data, expire=False)
-    except Exception:
-        logger.error(f"Copying repo {repo_info['id']} to {new_repo_id} failed.")
-        repo_problem(new_repo_id)
-        raise
-
-    return return_data
+    return {"name": sidetag_name, "id": sidetag_id, 'task_id': task_id}
 
 
 @export

From f6d67d8478cb6fee7db347e3709e130c5a0fd09a Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Jul 13 2023 13:19:14 +0000
Subject: [PATCH 3/4] make newRepo for sidetag configurable


---

diff --git a/plugins/cli/sidetag_cli.py b/plugins/cli/sidetag_cli.py
index a8e16f8..58137c5 100644
--- a/plugins/cli/sidetag_cli.py
+++ b/plugins/cli/sidetag_cli.py
@@ -9,7 +9,8 @@ from optparse import OptionParser
 
 import koji
 from koji.plugin import export_cli
-from koji_cli.lib import activate_session, arg_filter, watch_tasks
+from koji_cli.commands import anon_handle_wait_repo
+from koji_cli.lib import activate_session, arg_filter
 
 
 @export_cli
@@ -37,7 +38,7 @@ def handle_add_sidetag(options, session, args):
     if opts.suffix:
         kwargs['suffix'] = opts.suffix
     try:
-        info = session.createSideTag(basetag, **kwargs)
+        tag = session.createSideTag(basetag, **kwargs)
     except koji.ActionNotAllowed:
         parser.error("Policy violation")
     except koji.ParameterError as ex:
@@ -47,11 +48,13 @@ def handle_add_sidetag(options, session, args):
             raise
 
     if not opts.quiet:
-        print(info["name"])
+        print(tag["name"])
 
     if opts.wait:
-        return watch_tasks(session, [info['task_id']], quiet=opts.quiet,
-                           poll_interval=options.poll_interval, topurl=options.topurl)
+        args = ["--target", tag["name"]]
+        if opts.quiet:
+            args.append("--quiet")
+        anon_handle_wait_repo(options, session, args)
 
 
 @export_cli
diff --git a/plugins/hub/sidetag.conf b/plugins/hub/sidetag.conf
index d4e55a4..20a44c4 100644
--- a/plugins/hub/sidetag.conf
+++ b/plugins/hub/sidetag.conf
@@ -1,9 +1,15 @@
 [sidetag]
 # automatically remove sidetag on untagging last package
 remove_empty = off
+
 # potential suffixes for sidetag names
 # allowed_suffixes =
+
 # template for sidetag names. It must contain basetag and tag_id parts
 # if allowed_suffixes is not empty and suffix was requested, it will be added
 # as {name_template}-{suffix}. (percent-signs need to be escaped)
 # name_template = {basetag}-side-{tag_id}
+
+# Automaticaly trigger newRepo task for every new sidetag. Otherwise let kojira
+# prioritize these.
+# trigger_new_repo = False
diff --git a/plugins/hub/sidetag_hub.py b/plugins/hub/sidetag_hub.py
index 4ff1576..8932571 100644
--- a/plugins/hub/sidetag_hub.py
+++ b/plugins/hub/sidetag_hub.py
@@ -27,6 +27,7 @@ from kojihub.db import QueryProcessor, nextval
 CONFIG_FILE = "/etc/koji-hub/plugins/sidetag.conf"
 CONFIG = None
 ALLOWED_SUFFIXES = []
+TRIGGER_NEW_REPO = False
 
 
 def is_sidetag(taginfo, raise_error=False):
@@ -136,9 +137,13 @@ def createSideTag(basetag, debuginfo=False, suffix=None):
     )
     _create_build_target(sidetag_name, sidetag_id, sidetag_id)
 
-    # little higher priority than other newRepo tasks
-    args = koji.encode_args(sidetag_name, debuginfo=debuginfo)
-    task_id = make_task('newRepo', args, priority=14, channel='createrepo')
+
+    if TRIGGER_NEW_REPO:
+        # little higher priority than other newRepo tasks
+        args = koji.encode_args(sidetag_name, debuginfo=debuginfo)
+        task_id = make_task('newRepo', args, priority=14, channel='createrepo')
+    else:
+        task_id = None
 
     return {"name": sidetag_name, "id": sidetag_id, 'task_id': task_id}
 
@@ -367,3 +372,5 @@ if not CONFIG:
         NAME_TEMPLATE = CONFIG.get("sidetag", "name_template")
     else:
         NAME_TEMPLATE = '{basetag}-side-{tag_id}'
+    if CONFIG.has_option("sidetag", "trigger_new_repo"):
+        TRIGGER_NEW_REPO = CONFIG.getboolean("sidetag", "trigger_new_repo")

From 7afa72a60d4827235538e908762adcfc8abbeb05 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Jul 13 2023 15:03:23 +0000
Subject: [PATCH 4/4] fix tests


---

diff --git a/plugins/hub/sidetag_hub.py b/plugins/hub/sidetag_hub.py
index 8932571..6f3a730 100644
--- a/plugins/hub/sidetag_hub.py
+++ b/plugins/hub/sidetag_hub.py
@@ -137,7 +137,6 @@ def createSideTag(basetag, debuginfo=False, suffix=None):
     )
     _create_build_target(sidetag_name, sidetag_id, sidetag_id)
 
-
     if TRIGGER_NEW_REPO:
         # little higher priority than other newRepo tasks
         args = koji.encode_args(sidetag_name, debuginfo=debuginfo)
diff --git a/tests/test_plugins/test_sidetag_hub.py b/tests/test_plugins/test_sidetag_hub.py
index ebb87dc..8ef301d 100644
--- a/tests/test_plugins/test_sidetag_hub.py
+++ b/tests/test_plugins/test_sidetag_hub.py
@@ -51,7 +51,7 @@ class TestCreateSideTagHub(unittest.TestCase):
         self._create_tag.return_value = 12346
 
         ret = sidetag_hub.createSideTag('base_tag')
-        self.assertEqual(ret, {'name': sidetag_name, 'id': 12346})
+        self.assertEqual(ret, {'name': sidetag_name, 'id': 12346, 'task_id': None})
 
         self.get_user.assert_called_once_with(23, strict=True)
         self.get_tag.assert_called_once_with(self.basetag['name'], strict=True)
@@ -83,7 +83,7 @@ class TestCreateSideTagHub(unittest.TestCase):
         sidetag_hub.NAME_TEMPLATE = '{basetag}-sidetag-{tag_id}'
 
         ret = sidetag_hub.createSideTag('base_tag', debuginfo=True, suffix='suffix')
-        self.assertEqual(ret, {'name': sidetag_name, 'id': 12346})
+        self.assertEqual(ret, {'name': sidetag_name, 'id': 12346, 'task_id': None})
 
     def test_createsidetag_template_forbidden_suffix(self):
         sidetag_hub.ALLOWED_SUFFIXES = ['suffix', 'another']

