From b2c37233f1f5e5c69ee5134b6258634f7c2e4cdb Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Feb 07 2024 14:24:53 +0000
Subject: [PATCH 1/4] Track uploads in BuildRoot and log mock config


Related: https://pagure.io/koji/issue/3875

---

diff --git a/builder/kojid b/builder/kojid
index b4536dd..291dba7 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -199,6 +199,7 @@ class BuildRoot(object):
         self.logger = logging.getLogger("koji.build.buildroot")
         self.session = session
         self.options = options
+        self.logs = set()
         if len(args) + len(kwargs) == 1:
             # manage an existing mock buildroot
             self._load(*args, **kwargs)
@@ -346,6 +347,8 @@ class BuildRoot(object):
         with koji._open_text_file(configfile, 'wt') as fo:
             fo.write(output)
 
+        self.single_log(configfile, name='mock_config.log')
+
     def get_repo_dir(self):
         pathinfo = koji.PathInfo(topdir='')
         return pathinfo.repo(self.repoid, self.tag_name)
@@ -468,7 +471,6 @@ class BuildRoot(object):
         pid = os.fork()
         if pid:
             resultdir = self.resultdir()
-            uploadpath = self.getUploadPath()
             logs = {}
 
             ts_offsets = {}
@@ -550,7 +552,7 @@ class BuildRoot(object):
                             with koji._open_text_file(fpath, 'at') as ts_file:
                                 ts_file.write('%.0f %i\n' % (time.time(), position))
                             ts_offsets[fname] = position
-                    incremental_upload(self.session, fname, fd, uploadpath, logger=self.logger)
+                    self.incremental_log(fname, fd)
 
             # clean up and return exit status of command
             for (fname, (fd, inode, size, fpath)) in logs.items():
@@ -559,7 +561,7 @@ class BuildRoot(object):
                 if fname.endswith('-ts.log'):
                     # finish upload of ts.log as they could've been missed in
                     # last iteration
-                    incremental_upload(self.session, fname, fd, uploadpath, logger=self.logger)
+                    self.incremental_log(fname, fd)
                 fd.close()
             return status[1]
 
@@ -591,6 +593,17 @@ class BuildRoot(object):
         the hub."""
         return koji.pathinfo.taskrelpath(self.task_id)
 
+    def incremental_log(self, fname, fd):
+        ret = incremental_upload(self.session, fname, fd, self.getUploadPath(), logger=self.logger)
+        self.logs.add(fname)
+        return ret
+
+    def single_log(self, fname, name=None):
+        if name is None:
+            name = os.path.basename(fname)
+        self.session.uploadWrapper(fname, self.getUploadPath(), name=name)
+        self.logs.add(name)
+
     def init(self):
         rv = self.mock(['--init'])
 
@@ -1550,12 +1563,13 @@ class BuildArchTask(BaseBuildTask):
         resultdir = broot.resultdir()
         rpm_files = []
         srpm_files = []
-        log_files = []
+        log_files = list(broot.logs)
         unexpected = []
         for f in os.listdir(resultdir):
             # files here should have one of two extensions: .log and .rpm
-            if f[-4:] == ".log":
-                log_files.append(f)
+            if f[-4:] in (".log"):
+                pass
+                # should already be in log_files
             elif f[-8:] == ".src.rpm":
                 srpm_files.append(f)
             elif f[-4:] == ".rpm":
@@ -1563,12 +1577,6 @@ class BuildArchTask(BaseBuildTask):
             else:
                 unexpected.append(f)
 
-        if broot.workdir:
-            mocklog = 'mock_output.log'
-            f = os.path.join(broot.workdir, mocklog)
-            if os.path.exists(f):
-                log_files.append(os.path.basename(f))
-
         # for noarch rpms compute rpmdiff hash
         rpmdiff_hash = {self.id: {}}
         for rpmf in rpm_files:
@@ -1580,6 +1588,7 @@ class BuildArchTask(BaseBuildTask):
             log_name = 'noarch_rpmdiff.json'
             noarch_hash_path = os.path.join(broot.workdir, log_name)
             koji.dump_json(noarch_hash_path, rpmdiff_hash, indent=2, sort_keys=True)
+            self.uploadFile(noarch_hash_path)
             log_files.append(log_name)
 
         self.logger.debug("rpms: %r" % rpm_files)
@@ -1613,8 +1622,6 @@ class BuildArchTask(BaseBuildTask):
         else:
             ret['srpms'] = []
         ret['logs'] = ["%s/%s" % (uploadpath, f) for f in log_files]
-        if rpmdiff_hash[self.id]:
-            self.uploadFile(noarch_hash_path)
 
         ret['brootid'] = broot.id
 
@@ -1948,16 +1955,8 @@ class BuildMavenTask(BaseBuildTask):
                 self.uploadFile(os.path.join(outputdir, relpath, filename),
                                 relPath=relpath)
 
-        # Should only find log files in the mock result directory.
-        # Don't upload these log files, they've already been streamed
-        # the hub.
-        for filename in os.listdir(buildroot.resultdir()):
-            root, ext = os.path.splitext(filename)
-            if ext == '.log':
-                filepath = os.path.join(buildroot.resultdir(), filename)
-                if os.path.isfile(filepath) and os.stat(filepath).st_size > 0:
-                    # only files with content get uploaded to the hub
-                    logs.append(filename)
+        # Also include the logs already upload by BuildRoot
+        logs.extend(broot.logs)
 
         buildroot.expire()
 
@@ -2274,7 +2273,7 @@ class WrapperRPMTask(BaseBuildTask):
         srpm = None
         rpms = []
         specfile_name = os.path.basename(specfile)
-        logs = ['checkout.log', specfile_name]
+        logs = ['checkout.log', specfile_name] + list(buildroot.logs)
 
         for filename in os.listdir(resultdir):
             if filename.endswith('.src.rpm'):
@@ -2288,7 +2287,8 @@ class WrapperRPMTask(BaseBuildTask):
             elif filename.endswith('.rpm'):
                 rpms.append(filename)
             elif filename.endswith('.log'):
-                logs.append(filename)
+                pass
+                # already included in buildroot.logs
             else:
                 if self.new_build_id:
                     self.session.host.failBuild(self.id, self.new_build_id)
@@ -5129,14 +5129,13 @@ class RebuildSRPM(BaseBuildTask):
         self.uploadFile(srpm)
 
         brootid = broot.id
-        log_files = glob.glob('%s/*.log' % broot.resultdir())
+        log_files = list(broot.logs)
 
         broot.expire()
 
         return {
             'srpm': "%s/%s" % (uploadpath, srpm_name),
-            'logs': ["%s/%s" % (uploadpath, os.path.basename(f))
-                     for f in log_files],
+            'logs': ["%s/%s" % (uploadpath, f) for f in log_files],
             'brootid': brootid,
             'source': {
                 'source': os.path.basename(srpm),
@@ -5299,13 +5298,12 @@ class BuildSRPMFromSCMTask(BaseBuildTask):
         self.uploadFile(srpm)
 
         brootid = broot.id
-        log_files = glob.glob('%s/*.log' % broot.resultdir())
+        log_files = list(broot.logs)
 
         broot.expire()
 
         return {'srpm': "%s/%s" % (uploadpath, srpm_name),
-                'logs': ["%s/%s" % (uploadpath, os.path.basename(f))
-                         for f in log_files],
+                'logs': ["%s/%s" % (uploadpath, f) for f in log_files],
                 'brootid': brootid,
                 'source': source,
                 }

From 99d4752c2ce427a99c7bb397160c503850fa3704 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Feb 07 2024 14:24:55 +0000
Subject: [PATCH 2/4] rename var


---

diff --git a/builder/kojid b/builder/kojid
index 291dba7..b47cbda 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -598,10 +598,10 @@ class BuildRoot(object):
         self.logs.add(fname)
         return ret
 
-    def single_log(self, fname, name=None):
+    def single_log(self, localfile, name=None):
         if name is None:
-            name = os.path.basename(fname)
-        self.session.uploadWrapper(fname, self.getUploadPath(), name=name)
+            name = os.path.basename(localfile)
+        self.session.uploadWrapper(localfile, self.getUploadPath(), name=name)
         self.logs.add(name)
 
     def init(self):

From ffa93628d58660c028ae0dba3eb52a3219999566 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Feb 07 2024 14:24:55 +0000
Subject: [PATCH 3/4] also use in appliance/livecd/liveimage


---

diff --git a/builder/kojid b/builder/kojid
index b47cbda..040b776 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -3425,13 +3425,13 @@ class ApplianceTask(ImageTask):
         self.logger.debug('output: %s' % results)
         if len(results) == 0:
             raise koji.ApplianceError("Could not find image build results!")
+        logs = ['appliance.log', os.path.basename(ksfile), os.path.basename(kskoji)]
+        logs.extend(broot.logs)
         imgdata = {
             'arch': arch,
             'rootdev': self.getRootDevice(),
             'task_id': self.id,
-            'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log',
-                     'appliance.log', os.path.basename(ksfile),
-                     os.path.basename(kskoji)],
+            'logs': logs,
             'name': name,
             'version': version,
             'release': release
@@ -3597,13 +3597,13 @@ class LiveCDTask(ImageTask):
         self.uploadFile(manifest)
         self.uploadFile(isosrc, remoteName=isoname)
 
+        logs = ['livecd.log', os.path.basename(ksfile), os.path.basename(kskoji)]
+        logs.extend(broot.logs)
         imgdata = {'arch': arch,
                    'files': [isoname],
                    'rootdev': None,
                    'task_id': self.id,
-                   'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log',
-                            'livecd.log', os.path.basename(ksfile),
-                            os.path.basename(kskoji)],
+                   'logs': logs,
                    'name': name,
                    'version': version,
                    'release': release
@@ -3852,13 +3852,13 @@ class LiveMediaTask(ImageTask):
         self.logger.debug('uploading image: %s' % isosrc)
         self.uploadFile(isosrc, remoteName=isoname)
 
+        logs = ['livemedia-out.log', os.path.basename(ksfile), os.path.basename(kskoji)]
+        logs.extend(broot.logs)
         imgdata = {'arch': arch,
                    'files': [isoname],
                    'rootdev': None,
                    'task_id': self.id,
-                   'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log',
-                            'livemedia-out.log', os.path.basename(ksfile),
-                            os.path.basename(kskoji)],
+                   'logs': logs,
                    'name': name,
                    'version': version,
                    'release': release

From 4792b334191c493e9fa5f144a9bb467def8f553e Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Feb 28 2024 02:20:12 +0000
Subject: [PATCH 4/4] typo in BuildMavenTask


---

diff --git a/builder/kojid b/builder/kojid
index 040b776..fa36132 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -1956,7 +1956,7 @@ class BuildMavenTask(BaseBuildTask):
                                 relPath=relpath)
 
         # Also include the logs already upload by BuildRoot
-        logs.extend(broot.logs)
+        logs.extend(buildroot.logs)
 
         buildroot.expire()
 

