From 5bca4905184a3d5d1bde821fb339f0e959901291 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:46 +0000
Subject: [PATCH 1/92] wip


---

diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py
index 28d4418..bc14d40 100644
--- a/cli/koji_cli/commands.py
+++ b/cli/koji_cli/commands.py
@@ -7899,3 +7899,59 @@ def anon_handle_repoinfo(goptions, session, args):
             # repoID option added in 1.33
             if options.buildroots:
                 warn("--buildroots option is available with hub 1.33 or newer")
+
+
+def anon_handle_scheduler_info(goptions, session, args):
+    """[monitor] Show information about scheduling"""
+    usage = "usage: %prog schedulerinfo [options]"
+    parser = OptionParser(usage=get_usage_str(usage))
+    parser.add_option("-t", "--task", action="store", type=int, default=None,
+                      help="Limit data to given task id")
+    parser.add_option("--host", action="store", default=None,
+                      help="Limit data to given builder (name/id)")
+    parser.add_option("--state", action="store", type='str', default=None,
+                      choices=[x for x in koji.TASK_STATES.keys()],
+                      help="Limit data to task state")
+    (options, args) = parser.parse_args(args)
+    if len(args) > 0:
+        parser.error("This command takes no arguments")
+
+    ensure_connection(session, goptions)
+
+    host_id = None
+    if options.host:
+        try:
+            host_id = int(options.host)
+        except ValueError:
+            host_id = session.getHost(options.host, strict=True)['id']
+
+    if options.state:
+        state = koji.TASK_STATES[options.state]
+    else:
+        state = None
+
+    # get the data
+    runs = session.scheduler.getTaskRuns(taskID=options.task, hostID=host_id, state=state)
+    mask = '%(task_id)s\t%(host_id)s\t%(state)s\t%(create_time)s\t%(start_time)s\t%(end_time)s'
+    if not goptions.quiet:
+        header = mask % {
+            'task_id': 'Task',
+            'host_name': 'Host',
+            'state': 'State',
+            'create_time': 'Created',
+            'start_time': 'Started',
+            'end_time': 'Ended'
+        }
+        print(header)
+        print('-' * len(header))
+    for run in runs:
+        run['state'] = koji.TASK_STATES[runs['state']]
+        print(mask % run)
+
+    if host_id:
+        print('Host data for %s:' % options.host)
+        host_data = session.scheduler.getHostData(hostID=host_id)
+        if len(host_data) > 0:
+            print(host_data[0]['data'])
+        else:
+            print('-')
diff --git a/koji/__init__.py b/koji/__init__.py
index 4d0e7b2..6f1aa83 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -194,6 +194,8 @@ TASK_STATES = Enum((
     'CANCELED',
     'ASSIGNED',
     'FAILED',
+    'SCHEDULED',
+    'REFUSED',
 ))
 
 BUILD_STATES = Enum((
diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 20e72fb..3f568c0 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -95,6 +95,7 @@ from .db import (  # noqa: F401
 
 
 logger = logging.getLogger('koji.hub')
+sched_logger = scheduler.DBLogger()
 
 
 NUMERIC_TYPES = (int, float)
@@ -315,10 +316,12 @@ class Task(object):
         else:
             return None
 
-    def free(self):
+    def free(self, newstate=koji.TASK_STATES['FREE']):
         """Free a task"""
+        if newstate not in [koji.TASK_STATES['FREE'], koji.TASK_STATES['REFUSED']]:
+            raise koji.GenericError("Can't be called with other than FREE/REFUSED states")
         info = self.getInfo(request=True)
-        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES['FREE'])
+        self.runCallbacks('preTaskStateChange', info, 'state', newstate)
         self.runCallbacks('preTaskStateChange', info, 'host_id', None)
         # access checks should be performed by calling function
         query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(id)i'],
@@ -327,14 +330,13 @@ class Task(object):
         if not oldstate:
             raise koji.GenericError("No such task: %i" % self.id)
         if koji.TASK_STATES[oldstate] in ['CLOSED', 'CANCELED', 'FAILED']:
-            raise koji.GenericError("Cannot free task %i, state is %s" %
+            raise koji.GenericError("Cannot free/refuse task %i, state is %s" %
                                     (self.id, koji.TASK_STATES[oldstate]))
-        newstate = koji.TASK_STATES['FREE']
         newhost = None
         update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
                                  data={'state': newstate, 'host_id': newhost})
         update.execute()
-        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES['FREE'])
+        self.runCallbacks('postTaskStateChange', info, 'state', newstate)
         self.runCallbacks('postTaskStateChange', info, 'host_id', None)
         return True
 
@@ -14409,6 +14411,66 @@ class HostExports(object):
         task.assertHost(host.id)
         return task.setWeight(weight)
 
+    def setHostData(self, hostdata):
+        """Builder will update all its resources
+
+        Initial implementation contains:
+          - available task methods
+          - maxjobs
+          - host readiness
+        """
+        host = Host()
+        host.verify()
+        clauses = ['host_id = %(host_id)i']
+        values = {'host_id': host.id}
+        table = 'scheduler_host_data'
+        query = QueryProcessor(tables=[table], clauses=clauses, values=values,
+                               opts={'countOnly': True})
+        if query.singleValue() > 0:
+            update = UpdateProcessor(table=table, data={'data': hostdata},
+                                     clauses=clauses, values=values)
+            update.execute()
+        else:
+            insert = InsertProcessor(table=table, data={'data': hostdata},
+                                     clauses=clauses, values=values)
+            insert.execute()
+        sched_logger.debug(f"Updating host data with: {hostdata}",
+                           host_id=host.id, location='setHostData')
+
+    def getTasks(self):
+        host = Host()
+        host.verify()
+
+        query = QueryProcessor(
+            tables=['scheduler_task_runs'],
+            clauses=[
+                'host_id = %(host_id)s',
+                'state in %(states)s'
+            ],
+            values={
+                'host_id': host.id,
+                'states': [
+                    koji.TASK_STATES['SCHEDULED'],
+                    koji.TASK_STATES['ASSIGNED'],
+                ],
+            }
+        )
+        tasks = query.execute()
+        for task in tasks:
+            sched_logger.debug("Sending task", host_id=host.id, task_id=task['id'],
+                               location="getTasks")
+        return tasks
+
+    def refuseTask(self, task_id):
+        host = Host()
+        host.verify()
+
+        task = Task(task_id)
+        task.free(newstate=koji.TASK_STATES['REFUSED'])
+        sched_logger.warning("Refusing task", host_id=host.id, task_id=task_id,
+                             location="refuseTask")
+        return True
+
     def getHostTasks(self):
         host = Host()
         host.verify()
diff --git a/kojihub/kojixmlrpc.py b/kojihub/kojixmlrpc.py
index e9e0b3d..88f2f80 100644
--- a/kojihub/kojixmlrpc.py
+++ b/kojihub/kojixmlrpc.py
@@ -711,6 +711,7 @@ def setup_logging2(opts):
     log_handler.setFormatter(HubFormatter(opts['LogFormat']))
 
 
+    import scheduler
 def get_memory_usage():
     pagesize = resource.getpagesize()
     statm = [pagesize * int(y) // 1024
@@ -844,8 +845,10 @@ def get_registry(opts, plugins):
     registry = HandlerRegistry()
     functions = kojihub.RootExports()
     hostFunctions = kojihub.HostExports()
+    schedulerFunctions = scheduler.SchedulerExports()
     registry.register_instance(functions)
     registry.register_module(hostFunctions, "host")
+    registry.register_module(schedulerFunctions, "scheduler")
     registry.register_function(auth.login)
     registry.register_function(auth.sslLogin)
     registry.register_function(auth.logout)
diff --git a/schemas/schema.sql b/schemas/schema.sql
index 5fa6a22..cfe3932 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -983,6 +983,28 @@ CREATE TABLE rpm_checksum (
 ) WITHOUT OIDS;
 CREATE INDEX rpm_checksum_rpm_id ON rpm_checksum(rpm_id);
 
+
+-- scheduler tables
+CREATE TABLE scheduler_task_runs (
+        id SERIAL NOT NULL PRIMARY KEY,
+        task_id INTEGER REFERENCES task (id) NOT NULL,
+        host_id INTEGER REFERENCES host (id) NOT NULL,
+        state INTEGER NOT NULL,
+        create_time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        start_time TIMESTAMPTZ,
+        end_time TIMESTAMPTZ,
+) WITHOUT OIDS;
+CREATE INDEX scheduler_task_runs_task ON scheduler_task_runs(task_id);
+CREATE INDEX scheduler_task_runs_host ON scheduler_task_runs(host_id);
+CREATE INDEX scheduler_task_runs_state ON scheduler_task_runs(state);
+CREATE INDEX scheduler_task_runs_create_time ON scheduler_task_runs(create_time);
+
+CREATE TABLE scheduler_host_data (
+        host_id INTEGER REFERENCES host (id) PRIMARY KEY,
+        data JSONB,
+) WITHOUT OIDS;
+
+
 -- this table is used for locking, see db_lock()
 CREATE TABLE locks (
         name TEXT NOT NULL PRIMARY KEY

From edfe32c5c1ebaffd67e6a9ecde9e59bd94db8caf Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:46 +0000
Subject: [PATCH 2/92] replace old calls


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 3f568c0..5d424ed 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -2536,44 +2536,6 @@ def set_channel_enabled(channelname, enabled=True, comment=None):
     update.execute()
 
 
-def get_ready_hosts():
-    """Return information about hosts that are ready to build.
-
-    Hosts set the ready flag themselves
-    Note: We ignore hosts that are late checking in (even if a host
-        is busy with tasks, it should be checking in quite often).
-    """
-    query = QueryProcessor(
-        tables=['host'],
-        columns=['host.id', 'name', 'arches', 'task_load', 'capacity'],
-        aliases=['id', 'name', 'arches', 'task_load', 'capacity'],
-        clauses=[
-            'enabled IS TRUE',
-            'ready IS TRUE',
-            'expired IS FALSE',
-            'master IS NULL',
-            'active IS TRUE',
-            "sessions.update_time > NOW() - '5 minutes'::interval"
-        ],
-        joins=[
-            'sessions USING (user_id)',
-            'host_config ON host.id = host_config.host_id'
-        ]
-    )
-    hosts = query.execute()
-    for host in hosts:
-        query = QueryProcessor(
-            tables=['host_channels'],
-            columns=['channel_id'],
-            clauses=['host_id=%(id)s', 'active IS TRUE', 'enabled IS TRUE'],
-            joins=['channels ON host_channels.channel_id = channels.id'],
-            values=host
-        )
-        rows = query.execute()
-        host['channels'] = [row['channel_id'] for row in rows]
-    return hosts
-
-
 def get_all_arches():
     """Return a list of all (canonical) arches available from hosts"""
     ret = {}
@@ -2589,27 +2551,6 @@ def get_all_arches():
     return list(ret.keys())
 
 
-def get_active_tasks(host=None):
-    """Return data on tasks that are yet to be run"""
-    fields = ['id', 'state', 'channel_id', 'host_id', 'arch', 'method', 'priority', 'create_time']
-    values = dslice(koji.TASK_STATES, ('FREE', 'ASSIGNED'))
-    if host:
-        values['arches'] = host['arches'].split() + ['noarch']
-        values['channels'] = host['channels']
-        values['host_id'] = host['id']
-        clause = '(state = %(ASSIGNED)i AND host_id = %(host_id)i)'
-        if values['channels']:
-            clause += ''' OR (state = %(FREE)i AND arch IN %(arches)s \
-AND channel_id IN %(channels)s)'''
-        clauses = [clause]
-    else:
-        clauses = ['state IN (%(FREE)i,%(ASSIGNED)i)']
-    queryOpts = {'limit': 100, 'order': 'priority,create_time'}
-    query = QueryProcessor(columns=fields, tables=['task'], clauses=clauses,
-                           values=values, opts=queryOpts)
-    return query.execute()
-
-
 def get_task_descendents(task, childMap=None, request=False):
     if childMap is None:
         childMap = {}
@@ -14328,16 +14269,9 @@ class Host(object):
 
         This data is relatively small and the necessary load analysis is
         relatively complex, so we let the host machines crunch it."""
-        hosts = get_ready_hosts()
-        for host in hosts:
-            if host['id'] == self.id:
-                break
-        else:
-            # this host not in ready list
-            return [[], []]
-        # host is the host making the call
-        tasks = get_active_tasks(host)
-        return [hosts, tasks]
+        host = get_host(self.id)
+        tasks = scheduler.getTaskRuns(hostID=self.id)
+        return [[host], tasks]
 
     def isEnabled(self):
         """Return whether this host is enabled or not."""

From 64f949ebc342ac9df4bdee37c7abd04a2bceb6a1 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:46 +0000
Subject: [PATCH 3/92] partial: pulling work from other branch


---

diff --git a/schemas/schema.sql b/schemas/schema.sql
index cfe3932..007e75e 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -1005,6 +1005,18 @@ CREATE TABLE scheduler_host_data (
 ) WITHOUT OIDS;
 
 
+CREATE TABLE scheduler_log_messages (
+        id SERIAL NOT NULL PRIMARY KEY,
+        task_id INTEGER REFERENCES task (id),
+        host_id INTEGER REFERENCES host (id),
+        msg_time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        logger_name VARCHAR(200) NOT NULL,
+        level VARCHAR(10) NOT NULL,
+        location VARCHAR(200),
+        msg TEXT NOT NULL
+) WITHOUT OIDS;
+
+
 -- this table is used for locking, see db_lock()
 CREATE TABLE locks (
         name TEXT NOT NULL PRIMARY KEY

From 50bc4af985d442ed880f6037f8111605287fe473 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:46 +0000
Subject: [PATCH 4/92] ...


---

diff --git a/schemas/schema.sql b/schemas/schema.sql
index 007e75e..e778c21 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -999,12 +999,23 @@ CREATE INDEX scheduler_task_runs_host ON scheduler_task_runs(host_id);
 CREATE INDEX scheduler_task_runs_state ON scheduler_task_runs(state);
 CREATE INDEX scheduler_task_runs_create_time ON scheduler_task_runs(create_time);
 
+
 CREATE TABLE scheduler_host_data (
         host_id INTEGER REFERENCES host (id) PRIMARY KEY,
         data JSONB,
 ) WITHOUT OIDS;
 
 
+CREATE TABLE scheduler_map (
+        id SERIAL NOT NULL PRIMARY KEY,
+        task_id INTEGER REFERENCES task (id) NOT NULL,
+        host_id INTEGER REFERENCES host (id) NOT NULL,
+        runnable BOOLEAN NOT NULL,
+        refused BOOLEAN NOT NULL,
+        check_time TIMESTAMPTZ NOT NULL DEFAULT NOW()
+) WITHOUT OIDS;
+
+
 CREATE TABLE scheduler_log_messages (
         id SERIAL NOT NULL PRIMARY KEY,
         task_id INTEGER REFERENCES task (id),

From ee1df60db2a9496935925a4119a65cd733b04941 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:46 +0000
Subject: [PATCH 5/92] stub to let hub run


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 5d424ed..b7a4241 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -74,6 +74,7 @@ from koji.util import (
     multi_fnmatch,
     safer_move,
 )
+from . import scheduler
 from .auth import get_user_perms, get_user_groups
 from .db import (  # noqa: F401
     BulkInsertProcessor,
diff --git a/kojihub/kojixmlrpc.py b/kojihub/kojixmlrpc.py
index 88f2f80..f9000ca 100644
--- a/kojihub/kojixmlrpc.py
+++ b/kojihub/kojixmlrpc.py
@@ -41,6 +41,7 @@ from koji.server import ServerError, BadRequest, RequestTimeout
 from koji.xmlrpcplus import ExtendedMarshaller, Fault, dumps, getparser
 from . import auth
 from . import db
+from . import scheduler
 
 
 class Marshaller(ExtendedMarshaller):
@@ -711,7 +712,6 @@ def setup_logging2(opts):
     log_handler.setFormatter(HubFormatter(opts['LogFormat']))
 
 
-    import scheduler
 def get_memory_usage():
     pagesize = resource.getpagesize()
     statm = [pagesize * int(y) // 1024
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
new file mode 100644
index 0000000..7491021
--- /dev/null
+++ b/kojihub/scheduler.py
@@ -0,0 +1,9 @@
+# scheduler code goes here
+
+
+class DBLogger:
+    pass
+
+
+class SchedulerExports:
+    pass

From 6e9e1a8edbea60cab8b95605a47ac9625a1b7710 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:46 +0000
Subject: [PATCH 6/92] basic getTaskRuns function


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 7491021..57bf08b 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -1,5 +1,8 @@
 # scheduler code goes here
 
+from .kojihub import convert_value
+from .db import QueryProcessor
+
 
 class DBLogger:
     pass
@@ -7,3 +10,43 @@ class DBLogger:
 
 class SchedulerExports:
     pass
+
+
+def intlist(value):
+    """Cast value to a list of ints"""
+    if isinstance(value, (list, tuple)):
+        return [int(n) for n in value]
+    else:
+        return [int(value)]
+
+
+def getTaskRuns(taskID=None, hostID=None, state=None):
+    taskID = convert_value(taskID, cast=int, none_allowed=True)
+    hostID = convert_value(hostID, cast=int, none_allowed=True)
+    state = convert_value(state, cast=intlist, none_allowed=True)
+
+    fields = (
+        ('scheduler_task_runs.id', 'id'),
+        ('task_id', 'task_id'),
+        ('host_id', 'host_id'),
+        ('host.name', 'host_name'),
+        ("date_part('epoch', create_time)", 'create_ts'),
+        ("date_part('epoch', start_time)", 'start_ts'),
+        ("date_part('epoch', edn_time)", 'end_ts'),
+    )
+    fields, aliases = zip(*fields)
+
+    clauses = []
+    if taskID is not None:
+        clauses.append('task_id = %(taskID)s')
+    if hostID is not None:
+        clauses.append('host_id = %(hostID)s')
+    if state is not None:
+        clauses.append('host_id IN %(state)s')
+
+    query = QueryProcessor(
+        columns=fields, aliases=aliases, tables=['scheduler_task_runs'],
+        joins=['LEFT OUTER JOIN host on host_id=host.id'],
+        clauses=clauses, values=locals())
+
+    return query.execute()

From 4ca1c230dffd34c74f5ceff366810c05cdacc556 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:46 +0000
Subject: [PATCH 7/92] move convert_value to util lib


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index b7a4241..d31b23c 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -93,6 +93,7 @@ from .db import (  # noqa: F401
     nextval,
     currval,
 )
+from .util import convert_value
 
 
 logger = logging.getLogger('koji.hub')
@@ -111,36 +112,6 @@ def xform_user_krb(entry):
     return entry
 
 
-def convert_value(value, cast=None, message=None,
-                  exc_type=koji.ParameterError, none_allowed=False, check_only=False):
-    """Cast to another type with tailored exception
-
-    :param any value: tested object
-    :param type cast: To which type value should be cast
-    :param type exc_type: Raise this exception
-    :param bool none_allowed: Is None valid value?
-    :param check_only: Don't convert but raise an exception if type(value) != cast
-
-    :returns any value: returns converted value
-    """
-    if value is None:
-        if not none_allowed:
-            raise exc_type(message or f"Invalid type, expected type {cast}")
-        else:
-            return value
-    if check_only:
-        if not isinstance(value, cast):
-            raise exc_type(message or f"Invalid type for value '{value}': {type(value)}, "
-                                      f"expected type {cast}")
-    else:
-        try:
-            value = cast(value)
-        except (ValueError, TypeError):
-            raise exc_type(message or f"Invalid type for value '{value}': {type(value)}, "
-                                      f"expected type {cast}")
-    return value
-
-
 class Task(object):
     """A task for the build hosts"""
 
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 57bf08b..12a7495 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -1,7 +1,7 @@
 # scheduler code goes here
 
-from .kojihub import convert_value
 from .db import QueryProcessor
+from .util import convert_value
 
 
 class DBLogger:
diff --git a/kojihub/util.py b/kojihub/util.py
new file mode 100644
index 0000000..a874676
--- /dev/null
+++ b/kojihub/util.py
@@ -0,0 +1,33 @@
+import koji
+
+
+def convert_value(value, cast=None, message=None,
+                  exc_type=koji.ParameterError, none_allowed=False, check_only=False):
+    """Cast to another type with tailored exception
+
+    :param any value: tested object
+    :param type cast: To which type value should be cast
+    :param type exc_type: Raise this exception
+    :param bool none_allowed: Is None valid value?
+    :param check_only: Don't convert but raise an exception if type(value) != cast
+
+    :returns any value: returns converted value
+    """
+    if value is None:
+        if not none_allowed:
+            raise exc_type(message or f"Invalid type, expected type {cast}")
+        else:
+            return value
+    if check_only:
+        if not isinstance(value, cast):
+            raise exc_type(message or f"Invalid type for value '{value}': {type(value)}, "
+                                      f"expected type {cast}")
+    else:
+        try:
+            value = cast(value)
+        except (ValueError, TypeError):
+            raise exc_type(message or f"Invalid type for value '{value}': {type(value)}, "
+                                      f"expected type {cast}")
+    return value
+
+

From f0dcb423dba38f6f84be326c896bc54e1a54448a Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:46 +0000
Subject: [PATCH 8/92] typo


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 12a7495..10f96d3 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -32,7 +32,7 @@ def getTaskRuns(taskID=None, hostID=None, state=None):
         ('host.name', 'host_name'),
         ("date_part('epoch', create_time)", 'create_ts'),
         ("date_part('epoch', start_time)", 'start_ts'),
-        ("date_part('epoch', edn_time)", 'end_ts'),
+        ("date_part('epoch', end_time)", 'end_ts'),
     )
     fields, aliases = zip(*fields)
 
diff --git a/schemas/schema.sql b/schemas/schema.sql
index e778c21..2a555e7 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -992,7 +992,7 @@ CREATE TABLE scheduler_task_runs (
         state INTEGER NOT NULL,
         create_time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
         start_time TIMESTAMPTZ,
-        end_time TIMESTAMPTZ,
+        end_time TIMESTAMPTZ
 ) WITHOUT OIDS;
 CREATE INDEX scheduler_task_runs_task ON scheduler_task_runs(task_id);
 CREATE INDEX scheduler_task_runs_host ON scheduler_task_runs(host_id);
@@ -1002,7 +1002,7 @@ CREATE INDEX scheduler_task_runs_create_time ON scheduler_task_runs(create_time)
 
 CREATE TABLE scheduler_host_data (
         host_id INTEGER REFERENCES host (id) PRIMARY KEY,
-        data JSONB,
+        data JSONB
 ) WITHOUT OIDS;
 
 

From 3e7f9923ab17d40b876acafd5d76fd69c1425a83 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 9/92] return channels in getLoadData


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index d31b23c..e14fe85 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -14242,6 +14242,7 @@ class Host(object):
         This data is relatively small and the necessary load analysis is
         relatively complex, so we let the host machines crunch it."""
         host = get_host(self.id)
+        host['channels'] = [c['id'] for c in list_channels(hostID=self.id)]
         tasks = scheduler.getTaskRuns(hostID=self.id)
         return [[host], tasks]
 

From d012ddafcf397e95ba2c2a7ec08f5dfa87e4f71c Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 10/92] present task runs as assigned tasks to hosts using old api


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index e14fe85..96450ad 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -14244,6 +14244,11 @@ class Host(object):
         host = get_host(self.id)
         host['channels'] = [c['id'] for c in list_channels(hostID=self.id)]
         tasks = scheduler.getTaskRuns(hostID=self.id)
+        # for builders using this old api, we fake some of this data to get them to take the
+        # task runs assigned to them
+        for task in tasks:
+            task['state'] = koji.TASK_STATES['ASSIGNED']
+            task['host_id'] = self.id
         return [[host], tasks]
 
     def isEnabled(self):

From 745769de9e5ac3b0ec2857a0c64a875327c4d9f8 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 11/92] revert changes to freeTask


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 96450ad..e1ed2f5 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -288,12 +288,10 @@ class Task(object):
         else:
             return None
 
-    def free(self, newstate=koji.TASK_STATES['FREE']):
+    def free(self):
         """Free a task"""
-        if newstate not in [koji.TASK_STATES['FREE'], koji.TASK_STATES['REFUSED']]:
-            raise koji.GenericError("Can't be called with other than FREE/REFUSED states")
         info = self.getInfo(request=True)
-        self.runCallbacks('preTaskStateChange', info, 'state', newstate)
+        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES['FREE'])
         self.runCallbacks('preTaskStateChange', info, 'host_id', None)
         # access checks should be performed by calling function
         query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(id)i'],
@@ -302,13 +300,14 @@ class Task(object):
         if not oldstate:
             raise koji.GenericError("No such task: %i" % self.id)
         if koji.TASK_STATES[oldstate] in ['CLOSED', 'CANCELED', 'FAILED']:
-            raise koji.GenericError("Cannot free/refuse task %i, state is %s" %
+            raise koji.GenericError("Cannot free task %i, state is %s" %
                                     (self.id, koji.TASK_STATES[oldstate]))
+        newstate = koji.TASK_STATES['FREE']
         newhost = None
         update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
                                  data={'state': newstate, 'host_id': newhost})
         update.execute()
-        self.runCallbacks('postTaskStateChange', info, 'state', newstate)
+        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES['FREE'])
         self.runCallbacks('postTaskStateChange', info, 'host_id', None)
         return True
 
@@ -14377,6 +14376,8 @@ class HostExports(object):
         host = Host()
         host.verify()
 
+        # XXX
+
         task = Task(task_id)
         task.free(newstate=koji.TASK_STATES['REFUSED'])
         sched_logger.warning("Refusing task", host_id=host.id, task_id=task_id,

From 0430990092b568f60366ae8b36a03d884d105980 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 12/92] pull scheduler stub code from old branch


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 10f96d3..f635eea 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -50,3 +50,80 @@ def getTaskRuns(taskID=None, hostID=None, state=None):
         clauses=clauses, values=locals())
 
     return query.execute()
+
+
+def scheduler_map_task(taskinfo):
+    # map which hosts can take this task
+    # eventually this will involve more complex rules
+    q = QueryProcessor()
+    # select hosts matching arch and channel
+    hosts = q.execute()
+    u = InsertProcessor()
+
+
+class TaskScheduler(object):
+
+    def run(self):
+        if not self.get_lock():
+            # already running elsewhere
+            return
+
+        # get runs
+        fields = [
+                ('id', 'id'),
+                ('task_id', 'task_id'),
+                ('host_id', 'host_id'),
+                ('state', 'state'),
+                ("date_part('epoch', create_time)", 'create_ts'),
+                ("date_part('epoch', start_time)", 'start_ts'),
+                ("date_part('epoch', end_time)", 'end_ts')]
+        columns, aliases = zip(*fields.items())
+        query = QueryProcessor(columns = columns, aliases=aliases, tables=['scheduler_runs'])
+        runs = query.execute()
+        runs_by_task = {}
+        for run in runs:
+            runs_by_task.setdefault(run['task_id'], [])
+            runs_by_task[run['task_id']].append(run)
+
+        # get tasks
+        active_tasks = get_active_tasks()  # FREE and ASSIGNED, limit 100, priority ordered
+        # TODO need a better query, but this will do for now
+
+        # get hosts and bin them
+        hosts = get_ready_hosts()
+        hosts_by_bin = {}
+        for host in hosts:
+            host['_bins'] = []
+            for chan in host['channels']:
+                for arch in host['arches'].split() + ['noarch']:
+                    host_bin = "%s:%s" % (chan, arch)
+                    hosts_by_bin.setdefault(host_bin, []).append(host)
+                    host['_bins'].append(host_bin)
+
+        for task in active_tasks:
+            if task['state'] == koji.TASK_STATES['ASSIGNED']:
+                # TODO -- sort out our interaction with old school assignments
+                continue
+            have_run = False
+            task_runs = runs_by_task.get(task['id'], [])
+            for run in task_runs:
+                if run['state'] in OK_RUN_STATES:
+                    have_run = True
+                    break
+            if have_run:
+                continue
+            elif task_runs:
+                # TODO -- what to do about bad runs?
+            else:
+                # we need a run
+                # XXX need host
+                self.add_run(task, host)
+
+    def add_run(task, host):
+        insert = InsertProcessor('scheduler_runs')
+        insert.set(task_id=task['id'], host_id=host['id'], state=1)
+        insert.execute()
+
+    def get_lock(self):
+        # TODO
+        pass

From 3724d82edede8f8c9195c3997c20e5ec0f312355 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 13/92] tinkering


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index e1ed2f5..5d324d1 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -14243,6 +14243,11 @@ class Host(object):
         host = get_host(self.id)
         host['channels'] = [c['id'] for c in list_channels(hostID=self.id)]
         tasks = scheduler.getTaskRuns(hostID=self.id)
+        if not tasks:
+            # try running scheduler
+            if scheduler.TaskScheduler().run():
+                # check again
+                tasks = scheduler.getTaskRuns(hostID=self.id)
         # for builders using this old api, we fake some of this data to get them to take the
         # task runs assigned to them
         for task in tasks:
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index f635eea..787409b 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -30,6 +30,7 @@ def getTaskRuns(taskID=None, hostID=None, state=None):
         ('task_id', 'task_id'),
         ('host_id', 'host_id'),
         ('host.name', 'host_name'),
+        ('state', 'state'),
         ("date_part('epoch', create_time)", 'create_ts'),
         ("date_part('epoch', start_time)", 'start_ts'),
         ("date_part('epoch', end_time)", 'end_ts'),
@@ -49,7 +50,7 @@ def getTaskRuns(taskID=None, hostID=None, state=None):
         joins=['LEFT OUTER JOIN host on host_id=host.id'],
         clauses=clauses, values=locals())
 
-    return query.execute()
+    data = query.execute()
 
 
 def scheduler_map_task(taskinfo):
@@ -66,27 +67,16 @@ class TaskScheduler(object):
     def run(self):
         if not self.get_lock():
             # already running elsewhere
-            return
-
-        # get runs
-        fields = [
-                ('id', 'id'),
-                ('task_id', 'task_id'),
-                ('host_id', 'host_id'),
-                ('state', 'state'),
-                ("date_part('epoch', create_time)", 'create_ts'),
-                ("date_part('epoch', start_time)", 'start_ts'),
-                ("date_part('epoch', end_time)", 'end_ts')]
-        columns, aliases = zip(*fields.items())
-        query = QueryProcessor(columns = columns, aliases=aliases, tables=['scheduler_runs'])
-        runs = query.execute()
+            return False
+
+        runs = getTaskRuns()
         runs_by_task = {}
         for run in runs:
             runs_by_task.setdefault(run['task_id'], [])
             runs_by_task[run['task_id']].append(run)
 
         # get tasks
-        active_tasks = get_active_tasks()  # FREE and ASSIGNED, limit 100, priority ordered
+        active_tasks = self.get_tasks()
         # TODO need a better query, but this will do for now
 
         # get hosts and bin them
@@ -119,7 +109,39 @@ class TaskScheduler(object):
                 # XXX need host
                 self.add_run(task, host)
 
-    def add_run(task, host):
+        # indicate that scheduling ran
+        return True
+
+    def get_tasks(self):
+        pass
+
+
+    def get_task_data():
+        joins = ('LEFT OUTER JOIN scheduler_task_runs ON task_id = task.id')
+
+        fields = (
+            ('task.id', 'task_id'),
+            ('scheduler_task_runs.id', 'id'),
+            ('task_id', 'task_id'),
+            ('host_id', 'host_id'),
+            ('host.name', 'host_name'),
+            ('state', 'state'),
+            ("date_part('epoch', create_time)", 'create_ts'),
+            ("date_part('epoch', start_time)", 'start_ts'),
+            ("date_part('epoch', end_time)", 'end_ts'),
+        )
+        fields, aliases = zip(*fields)
+
+
+        query = QueryProcessor(
+            columns=fields, aliases=aliases, tables=['scheduler_task_runs'],
+            joins=['LEFT OUTER JOIN host on host_id=host.id'],
+            clauses=clauses, values=locals())
+
+        data = query.execute()
+
+
+    def add_run(self, task, host):
         insert = InsertProcessor('scheduler_runs')
         insert.set(task_id=task['id'], host_id=host['id'], state=1)
         insert.execute()

From 3b448320ad782a2539509b49f2b9308f9401ae42 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 14/92] ...


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 787409b..3bef6ef 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -75,9 +75,8 @@ class TaskScheduler(object):
             runs_by_task.setdefault(run['task_id'], [])
             runs_by_task[run['task_id']].append(run)
 
-        # get tasks
-        active_tasks = self.get_tasks()
-        # TODO need a better query, but this will do for now
+        # get tasks to schedule
+        tasks = self.get_tasks()
 
         # get hosts and bin them
         hosts = get_ready_hosts()
@@ -113,33 +112,33 @@ class TaskScheduler(object):
         return True
 
     def get_tasks(self):
-        pass
-
-
-    def get_task_data():
-        joins = ('LEFT OUTER JOIN scheduler_task_runs ON task_id = task.id')
+        """Get the tasks that need scheduling"""
 
         fields = (
             ('task.id', 'task_id'),
-            ('scheduler_task_runs.id', 'id'),
-            ('task_id', 'task_id'),
-            ('host_id', 'host_id'),
-            ('host.name', 'host_name'),
-            ('state', 'state'),
-            ("date_part('epoch', create_time)", 'create_ts'),
-            ("date_part('epoch', start_time)", 'start_ts'),
-            ("date_part('epoch', end_time)", 'end_ts'),
+            ('task.state', 'state'),
+            ('channel_id', 'channel_id'),
+            ('task.host_id', 'host_id'),
+            ('arch', 'arch'),
+            ('method', 'method'),
+            ('priority', 'priority'),
+            ("date_part('epoch', task.create_time)", 'create_ts'),
+            ('scheduler_task_runs.id', 'run_id'),
         )
         fields, aliases = zip(*fields)
 
+        values = {'states': [koji.TASK_STATES[n] for n in ('FREE', 'ASSIGNED')]}
 
         query = QueryProcessor(
-            columns=fields, aliases=aliases, tables=['scheduler_task_runs'],
-            joins=['LEFT OUTER JOIN host on host_id=host.id'],
-            clauses=clauses, values=locals())
-
-        data = query.execute()
+            columns=fields, aliases=aliases, tables=['task'],
+            joins=('LEFT OUTER JOIN scheduler_task_runs ON task_id = task.id'),
+            clauses=('task.state IN %(states)s', 'run_id IS NULL'),
+            # XXX these clauses are not enough
+            values=values,
+            opts={'order': 'priority,create_time'}
+        )
 
+        return query.execute()
 
     def add_run(self, task, host):
         insert = InsertProcessor('scheduler_runs')

From 1cffaefc45264b7d80388c42e554cdef31da09a8 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 15/92] typo


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 3bef6ef..c095907 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -103,6 +103,7 @@ class TaskScheduler(object):
                 continue
             elif task_runs:
                 # TODO -- what to do about bad runs?
+                pass
             else:
                 # we need a run
                 # XXX need host

From e70eeac42bcaa97d1c48fb0aeb3be0e67034e3d7 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 16/92] more work


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index c095907..8d29d72 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -76,10 +76,15 @@ class TaskScheduler(object):
             runs_by_task[run['task_id']].append(run)
 
         # get tasks to schedule
-        tasks = self.get_tasks()
+        tasks = self.get_free_tasks()
+        tasks_by_bin = {}
+        for task in tasks:
+            tbin = '%(channel_id)s:%(arch)s' % task
+            task['_bin'] = tbin
+            tasks_by_bin.setdefault(tbin, []).append(task)
 
         # get hosts and bin them
-        hosts = get_ready_hosts()
+        hosts = self.get_ready_hosts()
         hosts_by_bin = {}
         for host in hosts:
             host['_bins'] = []
@@ -89,10 +94,34 @@ class TaskScheduler(object):
                     hosts_by_bin.setdefault(host_bin, []).append(host)
                     host['_bins'].append(host_bin)
 
-        for task in active_tasks:
-            if task['state'] == koji.TASK_STATES['ASSIGNED']:
-                # TODO -- sort out our interaction with old school assignments
-                continue
+        # order bins by available host capacity
+        order = []
+        for _bin in hosts_by_bin:
+            hosts = hosts_by_bin[_bin]
+            avail = sum([min(0, h['capacity'] - h['task_load']) for h in hosts])
+            order.append((avail, _bin))
+        order.sort()
+
+        # note bin demand for each host
+        for n, (avail, _bin) in enumerate(order):
+            rank = float(n) / len(order)
+            for host in hosts_by_bin[_bin]:
+                host.setdefault('_rank', rank)
+                # so host rank is set by the most contentious bin it covers
+                # TODO - we could be smarter here, but it's a start
+
+        # sort binned hosts by rank
+        for _bin in hosts_by_bin:
+            hosts = hosts_by_bin[_bin]
+            hosts.sort(key=lambda h: h._rank, reverse=True)
+            # hosts with least contention first
+
+        # tasks are already in priority order
+        for task in tasks:
+            hosts = hosts_by_bin[task['_bin']]
+            # these are the hosts that _can_ take this task
+            # TODO - update host ranks as we go
+
             have_run = False
             task_runs = runs_by_task.get(task['id'], [])
             for run in task_runs:
@@ -112,7 +141,7 @@ class TaskScheduler(object):
         # indicate that scheduling ran
         return True
 
-    def get_tasks(self):
+    def get_free_tasks(self):
         """Get the tasks that need scheduling"""
 
         fields = (
@@ -124,23 +153,71 @@ class TaskScheduler(object):
             ('method', 'method'),
             ('priority', 'priority'),
             ("date_part('epoch', task.create_time)", 'create_ts'),
-            ('scheduler_task_runs.id', 'run_id'),
+            # ('scheduler_task_runs.id', 'run_id'),
         )
         fields, aliases = zip(*fields)
 
-        values = {'states': [koji.TASK_STATES[n] for n in ('FREE', 'ASSIGNED')]}
+        values = {'states': [koji.TASK_STATES[n] for n in ('FREE',)]}
 
         query = QueryProcessor(
             columns=fields, aliases=aliases, tables=['task'],
-            joins=('LEFT OUTER JOIN scheduler_task_runs ON task_id = task.id'),
-            clauses=('task.state IN %(states)s', 'run_id IS NULL'),
-            # XXX these clauses are not enough
+            # joins=('LEFT OUTER JOIN scheduler_task_runs ON task_id = task.id'),
+            # clauses=('task.state IN %(states)s', 'run_id IS NULL'),
+            clauses=('task.state IN %(states)s',),
             values=values,
-            opts={'order': 'priority,create_time'}
+            opts={'order': 'priority,create_time'},
+            # scheduler order
+            # lower priority numbers take precedence, like posix process priority
+            # at a given priority, earlier creation times take precedence
         )
 
         return query.execute()
 
+    def get_ready_hosts():
+        """Query hosts that are ready to build"""
+
+        fields = (
+            ('host.id', 'id'),
+            ('host.name', 'name'),
+            ('host.task_load', 'task_load'),
+            ('host_config.arches', 'arches'),
+            ('host_config.capacity', 'capacity'),
+            ("date_part('epoch', sessions.update_time)", 'update_ts'),
+        )
+        fields, aliases = zip(*fields)
+
+        query = QueryProcessor(
+            tables=['host'],
+            columns=fields,
+            aliases=aliases,
+            clauses=[
+                'enabled IS TRUE',
+                'ready IS TRUE',
+                'expired IS FALSE',
+                'master IS NULL',
+                'active IS TRUE',
+                "update_time > NOW() - '5 minutes'::interval"
+            ],
+            joins=[
+                'sessions USING (user_id)',
+                'host_config ON host.id = host_config.host_id'
+            ]
+        )
+
+        hosts = query.execute()
+        for host in hosts:
+            query = QueryProcessor(
+                tables=['host_channels'],
+                columns=['channel_id'],
+                clauses=['host_id=%(id)s', 'active IS TRUE', 'enabled IS TRUE'],
+                joins=['channels ON host_channels.channel_id = channels.id'],
+                values=host
+            )
+            rows = query.execute()
+            host['channels'] = [row['channel_id'] for row in rows]
+
+        return hosts
+
     def add_run(self, task, host):
         insert = InsertProcessor('scheduler_runs')
         insert.set(task_id=task['id'], host_id=host['id'], state=1)

From d0d774e70367a6e1e11bd76b933d21a4fdc814a0 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 17/92] ...


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 8d29d72..9671ff3 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -64,17 +64,18 @@ def scheduler_map_task(taskinfo):
 
 class TaskScheduler(object):
 
+    def __init__(self):
+        self.hosts_by_bin = None
+
     def run(self):
         if not self.get_lock():
             # already running elsewhere
             return False
 
-        runs = getTaskRuns()
-        runs_by_task = {}
-        for run in runs:
-            runs_by_task.setdefault(run['task_id'], [])
-            runs_by_task[run['task_id']].append(run)
+        self.do_schedule()
+        return True
 
+    def do_schedule(self):
         # get tasks to schedule
         tasks = self.get_free_tasks()
         tasks_by_bin = {}
@@ -138,9 +139,6 @@ class TaskScheduler(object):
                 # XXX need host
                 self.add_run(task, host)
 
-        # indicate that scheduling ran
-        return True
-
     def get_free_tasks(self):
         """Get the tasks that need scheduling"""
 
diff --git a/schemas/schema.sql b/schemas/schema.sql
index 2a555e7..ee62bc4 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -1012,6 +1012,7 @@ CREATE TABLE scheduler_map (
         host_id INTEGER REFERENCES host (id) NOT NULL,
         runnable BOOLEAN NOT NULL,
         refused BOOLEAN NOT NULL,
+        msg TEXT,
         check_time TIMESTAMPTZ NOT NULL DEFAULT NOW()
 ) WITHOUT OIDS;
 

From 63a8e3dcbff2a4e8b07266442dc960969d7b8c16 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 18/92] cleanup


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 9671ff3..9759bb4 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -1,5 +1,6 @@
 # scheduler code goes here
 
+import koji
 from .db import QueryProcessor
 from .util import convert_value
 
@@ -98,7 +99,7 @@ class TaskScheduler(object):
         # order bins by available host capacity
         order = []
         for _bin in hosts_by_bin:
-            hosts = hosts_by_bin[_bin]
+            hosts = hosts_by_bin.get(_bin, [])
             avail = sum([min(0, h['capacity'] - h['task_load']) for h in hosts])
             order.append((avail, _bin))
         order.sort()
@@ -106,7 +107,7 @@ class TaskScheduler(object):
         # note bin demand for each host
         for n, (avail, _bin) in enumerate(order):
             rank = float(n) / len(order)
-            for host in hosts_by_bin[_bin]:
+            for host in hosts_by_bin.get(_bin, []):
                 host.setdefault('_rank', rank)
                 # so host rank is set by the most contentious bin it covers
                 # TODO - we could be smarter here, but it's a start
@@ -119,25 +120,11 @@ class TaskScheduler(object):
 
         # tasks are already in priority order
         for task in tasks:
-            hosts = hosts_by_bin[task['_bin']]
+            hosts = hosts_by_bin.get(task['_bin'], [])
             # these are the hosts that _can_ take this task
             # TODO - update host ranks as we go
+            # TODO - pick a host and assign
 
-            have_run = False
-            task_runs = runs_by_task.get(task['id'], [])
-            for run in task_runs:
-                if run['state'] in OK_RUN_STATES:
-                    have_run = True
-                    break
-            if have_run:
-                continue
-            elif task_runs:
-                # TODO -- what to do about bad runs?
-                pass
-            else:
-                # we need a run
-                # XXX need host
-                self.add_run(task, host)
 
     def get_free_tasks(self):
         """Get the tasks that need scheduling"""
@@ -163,7 +150,7 @@ class TaskScheduler(object):
             # clauses=('task.state IN %(states)s', 'run_id IS NULL'),
             clauses=('task.state IN %(states)s',),
             values=values,
-            opts={'order': 'priority,create_time'},
+            opts={'order': 'priority,create_ts'},
             # scheduler order
             # lower priority numbers take precedence, like posix process priority
             # at a given priority, earlier creation times take precedence
@@ -171,7 +158,7 @@ class TaskScheduler(object):
 
         return query.execute()
 
-    def get_ready_hosts():
+    def get_ready_hosts(self):
         """Query hosts that are ready to build"""
 
         fields = (
@@ -223,4 +210,4 @@ class TaskScheduler(object):
 
     def get_lock(self):
         # TODO
-        pass
+        return True  # XXX

From 026e74472f7c66526f4b71b21771cea3d7be914c Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 19/92] table lock


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 9759bb4..f366593 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -1,8 +1,9 @@
-# scheduler code goes here
+import psycopg2
 
 import koji
 from .db import QueryProcessor
 from .util import convert_value
+from koji.context import context
 
 
 class DBLogger:
@@ -209,5 +210,11 @@ class TaskScheduler(object):
         insert.execute()
 
     def get_lock(self):
-        # TODO
-        return True  # XXX
+        c = context.cnx.cursor()
+        try:
+            c.execute('LOCK TABLE scheduler_map IN EXCLUSIVE MODE NOWAIT', log_errors=False)
+            # This allows parallel reads, but nothing else
+            # Note that even though we don't log the errors, postgres itself might
+        except psycopg2.OperationalError:
+            return False
+        return True

From 67667447044d03626b61671cb46678896abb332d Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 20/92] XXX TEST


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 5d324d1..0606cc4 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -10017,6 +10017,9 @@ def _delete_event_id():
 class RootExports(object):
     '''Contains functions that are made available via XMLRPC'''
 
+    def TEST(self):
+        scheduler.TaskScheduler().run()
+
     def restartHosts(self, priority=5, options=None):
         """Spawns restartHosts task
 

From 2c7d388164416dd82399439547fdc81ca887a90f Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 21/92] use db_lock in scheduler


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index f366593..fd9300c 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -1,7 +1,7 @@
 import psycopg2
 
 import koji
-from .db import QueryProcessor
+from .db import QueryProcessor, db_lock
 from .util import convert_value
 from koji.context import context
 
@@ -70,7 +70,7 @@ class TaskScheduler(object):
         self.hosts_by_bin = None
 
     def run(self):
-        if not self.get_lock():
+        if not db_lock('scheduler', wait=False):
             # already running elsewhere
             return False
 
@@ -208,13 +208,3 @@ class TaskScheduler(object):
         insert = InsertProcessor('scheduler_runs')
         insert.set(task_id=task['id'], host_id=host['id'], state=1)
         insert.execute()
-
-    def get_lock(self):
-        c = context.cnx.cursor()
-        try:
-            c.execute('LOCK TABLE scheduler_map IN EXCLUSIVE MODE NOWAIT', log_errors=False)
-            # This allows parallel reads, but nothing else
-            # Note that even though we don't log the errors, postgres itself might
-        except psycopg2.OperationalError:
-            return False
-        return True

From e500a3cda741adaa6987fcef5e17a26fd0fe228a Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 22/92] tweak clause order


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index fd9300c..df172ca 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -177,12 +177,12 @@ class TaskScheduler(object):
             columns=fields,
             aliases=aliases,
             clauses=[
-                'enabled IS TRUE',
-                'ready IS TRUE',
-                'expired IS FALSE',
-                'master IS NULL',
-                'active IS TRUE',
-                "update_time > NOW() - '5 minutes'::interval"
+                'host.ready IS TRUE',
+                'host_config.enabled IS TRUE',
+                'host_config.active IS TRUE',
+                'sessions.expired IS FALSE',
+                'sessions.master IS NULL',
+                "sessions.update_time > NOW() - '5 minutes'::interval"
             ],
             joins=[
                 'sessions USING (user_id)',

From 855d6e3bb40223f0fd2acfc69d8b4be93a97be85 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 23/92] update scheduler runs as tasks are updated


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 0606cc4..af567bb 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -259,6 +259,16 @@ class Task(object):
         if state == koji.TASK_STATES['OPEN']:
             update.rawset(start_time='NOW()')
         update.execute()
+        if state == koji.TASK_STATES['OPEN']:
+            # also update run entry
+            update2 = UpdateProcessor(
+                'scheduler_runs',
+                clauses=['task_id=%(task_id)s', 'host_id=%(host_id)s'],
+                values=locals(),
+                data={'state': state},
+                rawdata={'start_time': 'NOW()'},
+            )
+            update2.execute()
         self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES[newstate])
         self.runCallbacks('postTaskStateChange', info, 'host_id', host_id)
         return True
@@ -360,6 +370,17 @@ class Task(object):
                                  data={'result': info['result'], 'state': state},
                                  rawdata={'completion_time': 'NOW()'})
         update.execute()
+
+        # also update run entry
+        update2 = UpdateProcessor(
+            'scheduler_runs',
+            clauses=['task_id=%(task_id)s', 'host_id=%(host_id)s'],
+            values=locals(),
+            data={'state': state},
+            rawdata={'end_time': 'NOW()'},
+        )
+        update2.execute()
+
         self.runCallbacks('postTaskStateChange', info, 'state', state)
         self.runCallbacks('postTaskStateChange', info, 'completion_ts', now)
 
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index df172ca..988603a 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -206,5 +206,5 @@ class TaskScheduler(object):
 
     def add_run(self, task, host):
         insert = InsertProcessor('scheduler_runs')
-        insert.set(task_id=task['id'], host_id=host['id'], state=1)
+        insert.set(task_id=task['id'], host_id=host['id'], state=koji.TASK_STATES['ASSIGNED'])
         insert.execute()

From bb1a2d3fb3c3f49bc0c101642d880eb26410faa0 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 24/92] ...


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 988603a..d997aa9 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -68,6 +68,9 @@ class TaskScheduler(object):
 
     def __init__(self):
         self.hosts_by_bin = None
+        self.tasks_by_bin = None
+        self.active_tasks = None
+        self.maxjobs = 15  # XXX
 
     def run(self):
         if not db_lock('scheduler', wait=False):
@@ -75,32 +78,26 @@ class TaskScheduler(object):
             return False
 
         self.do_schedule()
+        # TODO clean up bad data (e.g. active tasks with no host)
+        # TODO check for runs that aren't getting picked up
+
         return True
 
-    def do_schedule(self):
-        # get tasks to schedule
-        tasks = self.get_free_tasks()
-        tasks_by_bin = {}
-        for task in tasks:
-            tbin = '%(channel_id)s:%(arch)s' % task
-            task['_bin'] = tbin
-            tasks_by_bin.setdefault(tbin, []).append(task)
+    def get_runs(self):
+        runs = getTaskRuns()
+        runs_by_task = {}
+        for run in runs:
+            runs_by_task.setdefault(run['task_id'], [])
+            runs_by_task[run['task_id']].append(run)
 
-        # get hosts and bin them
-        hosts = self.get_ready_hosts()
-        hosts_by_bin = {}
-        for host in hosts:
-            host['_bins'] = []
-            for chan in host['channels']:
-                for arch in host['arches'].split() + ['noarch']:
-                    host_bin = "%s:%s" % (chan, arch)
-                    hosts_by_bin.setdefault(host_bin, []).append(host)
-                    host['_bins'].append(host_bin)
+    def do_schedule(self):
+        self.get_tasks()
+        self.get_hosts()
 
         # order bins by available host capacity
         order = []
-        for _bin in hosts_by_bin:
-            hosts = hosts_by_bin.get(_bin, [])
+        for _bin in self.hosts_by_bin:
+            hosts = self.hosts_by_bin.get(_bin, [])
             avail = sum([min(0, h['capacity'] - h['task_load']) for h in hosts])
             order.append((avail, _bin))
         order.sort()
@@ -114,25 +111,26 @@ class TaskScheduler(object):
                 # TODO - we could be smarter here, but it's a start
 
         # sort binned hosts by rank
-        for _bin in hosts_by_bin:
-            hosts = hosts_by_bin[_bin]
+        for _bin in self.hosts_by_bin:
+            hosts = self.hosts_by_bin[_bin]
             hosts.sort(key=lambda h: h._rank, reverse=True)
             # hosts with least contention first
 
         # tasks are already in priority order
         for task in tasks:
-            hosts = hosts_by_bin.get(task['_bin'], [])
+            hosts = self.hosts_by_bin.get(task['_bin'], [])
             # these are the hosts that _can_ take this task
             # TODO - update host ranks as we go
             # TODO - pick a host and assign
 
 
-    def get_free_tasks(self):
-        """Get the tasks that need scheduling"""
+    def get_tasks(self):
+        """Get the task data that we need for scheduling"""
 
         fields = (
             ('task.id', 'task_id'),
             ('task.state', 'state'),
+            ('task.waiting', 'waiting'),
             ('channel_id', 'channel_id'),
             ('task.host_id', 'host_id'),
             ('arch', 'arch'),
@@ -143,21 +141,55 @@ class TaskScheduler(object):
         )
         fields, aliases = zip(*fields)
 
-        values = {'states': [koji.TASK_STATES[n] for n in ('FREE',)]}
+        values = {'states': [koji.TASK_STATES[n] for n in ('ASSIGNED', 'OPEN')]}
 
         query = QueryProcessor(
             columns=fields, aliases=aliases, tables=['task'],
-            # joins=('LEFT OUTER JOIN scheduler_task_runs ON task_id = task.id'),
-            # clauses=('task.state IN %(states)s', 'run_id IS NULL'),
-            clauses=('task.state IN %(states)s',),
+            clauses=('task.state IN %(states)s',
+                     'task.host_id IS NOT NULL',  # should always be set, but...
+                    ),
             values=values,
-            opts={'order': 'priority,create_ts'},
+        )
+        active_tasks = query.execute()
+
+        values = {'state': koji.TASK_STATES['FREE']}
+        query = QueryProcessor(
+            columns=fields, aliases=aliases, tables=['task'],
+            clauses=('task.state = %(state)s',),
+            values=values,
+            opts={'order': 'priority,create_ts', 'limit': 1000},  # TODO config
             # scheduler order
             # lower priority numbers take precedence, like posix process priority
             # at a given priority, earlier creation times take precedence
         )
+        free_tasks = query.execute()
+
+        tasks_by_bin = {}
+        for task in free_tasks:
+            tbin = '%(channel_id)s:%(arch)s' % task
+            task['_bin'] = tbin
+            tasks_by_bin.setdefault(tbin, []).append(task)
+
+        for task in active_tasks:
+            tbin = '%(channel_id)s:%(arch)s' % task
+            task['_bin'] = tbin
+
+        self.tasks_by_bin = tasks_by_bin
+        self.active_tasks = active_tasks
+
+    def get_hosts(self):
+        # get hosts and bin them
+        hosts = self.get_ready_hosts()
+        hosts_by_bin = {}
+        for host in hosts:
+            host['_bins'] = []
+            for chan in host['channels']:
+                for arch in host['arches'].split() + ['noarch']:
+                    host_bin = "%s:%s" % (chan, arch)
+                    hosts_by_bin.setdefault(host_bin, []).append(host)
+                    host['_bins'].append(host_bin)
 
-        return query.execute()
+        self.hosts_by_bin = hosts_by_bin
 
     def get_ready_hosts(self):
         """Query hosts that are ready to build"""
diff --git a/schemas/schema.sql b/schemas/schema.sql
index ee62bc4..f97ac2e 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -1000,6 +1000,13 @@ CREATE INDEX scheduler_task_runs_state ON scheduler_task_runs(state);
 CREATE INDEX scheduler_task_runs_create_time ON scheduler_task_runs(create_time);
 
 
+CREATE TABLE scheduler_queue (
+        task_id INTEGER REFERENCES task (id) NOT NULL PRIMARY KEY,
+        groomed BOOLEAN NOT NULL DEFAULT FALSE,
+        run_id INTEGER REFERENCES scheduler_task_run(id)
+) WITHOUT OIDS;
+
+
 CREATE TABLE scheduler_host_data (
         host_id INTEGER REFERENCES host (id) PRIMARY KEY,
         data JSONB

From 6750eea2a1919413401f3a9354a8f65841ed0a88 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 25/92] ...


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index d997aa9..218b276 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -1,3 +1,4 @@
+import logging
 import psycopg2
 
 import koji
@@ -6,6 +7,10 @@ from .util import convert_value
 from koji.context import context
 
 
+logger = logging.getLogger('koji.scheduler')
+# TODO set up db logging
+
+
 class DBLogger:
     pass
 
@@ -68,6 +73,7 @@ class TaskScheduler(object):
 
     def __init__(self):
         self.hosts_by_bin = None
+        self.hosts = None
         self.tasks_by_bin = None
         self.active_tasks = None
         self.maxjobs = 15  # XXX
@@ -94,18 +100,41 @@ class TaskScheduler(object):
         self.get_tasks()
         self.get_hosts()
 
+        # calculate host load and task count
+        for task in self.active_tasks:
+            # for now, we mirror what kojid updateTasks has been doing
+            host = self.hosts.get(task['host_id'])
+            if not host:
+                # not showing as ready
+                # TODO log and deal with this condition
+                continue
+            host.setdefault('_load', 0.0)
+            if not task['waiting']:
+                host['_load'] += task['weight']
+            host.setdefault('_ntasks', 0)
+            host['_ntasks'] += 1
+
+        for host in self.hosts.values():
+            host.setdefault('_load', 0.0)
+            # temporary test code
+            logger.info(f'Host: {host}')
+            ldiff = host['task_load'] - host['_load']
+            if abs(ldiff) > 0.01:
+                # this is expected in a number of cases, just observing
+                logger.info(f'Host load differs by {ldiff:.2f}: {host}')
+
         # order bins by available host capacity
         order = []
         for _bin in self.hosts_by_bin:
             hosts = self.hosts_by_bin.get(_bin, [])
-            avail = sum([min(0, h['capacity'] - h['task_load']) for h in hosts])
+            avail = sum([min(0, h['capacity'] - h['_load']) for h in hosts])
             order.append((avail, _bin))
         order.sort()
 
         # note bin demand for each host
         for n, (avail, _bin) in enumerate(order):
             rank = float(n) / len(order)
-            for host in hosts_by_bin.get(_bin, []):
+            for host in self.hosts_by_bin.get(_bin, []):
                 host.setdefault('_rank', rank)
                 # so host rank is set by the most contentious bin it covers
                 # TODO - we could be smarter here, but it's a start
@@ -113,11 +142,11 @@ class TaskScheduler(object):
         # sort binned hosts by rank
         for _bin in self.hosts_by_bin:
             hosts = self.hosts_by_bin[_bin]
-            hosts.sort(key=lambda h: h._rank, reverse=True)
+            hosts.sort(key=lambda h: h['_rank'], reverse=True)
             # hosts with least contention first
 
         # tasks are already in priority order
-        for task in tasks:
+        for task in self.free_tasks:
             hosts = self.hosts_by_bin.get(task['_bin'], [])
             # these are the hosts that _can_ take this task
             # TODO - update host ranks as we go
@@ -131,6 +160,7 @@ class TaskScheduler(object):
             ('task.id', 'task_id'),
             ('task.state', 'state'),
             ('task.waiting', 'waiting'),
+            ('task.weight', 'weight'),
             ('channel_id', 'channel_id'),
             ('task.host_id', 'host_id'),
             ('arch', 'arch'),
@@ -164,25 +194,24 @@ class TaskScheduler(object):
         )
         free_tasks = query.execute()
 
-        tasks_by_bin = {}
         for task in free_tasks:
             tbin = '%(channel_id)s:%(arch)s' % task
             task['_bin'] = tbin
-            tasks_by_bin.setdefault(tbin, []).append(task)
 
         for task in active_tasks:
             tbin = '%(channel_id)s:%(arch)s' % task
             task['_bin'] = tbin
 
-        self.tasks_by_bin = tasks_by_bin
+        self.free_tasks = free_tasks
         self.active_tasks = active_tasks
 
     def get_hosts(self):
         # get hosts and bin them
-        hosts = self.get_ready_hosts()
         hosts_by_bin = {}
-        for host in hosts:
+        hosts_by_id = {}
+        for host in self.get_ready_hosts():
             host['_bins'] = []
+            hosts_by_id[host['id']] = host
             for chan in host['channels']:
                 for arch in host['arches'].split() + ['noarch']:
                     host_bin = "%s:%s" % (chan, arch)
@@ -190,6 +219,7 @@ class TaskScheduler(object):
                     host['_bins'].append(host_bin)
 
         self.hosts_by_bin = hosts_by_bin
+        self.hosts = hosts_by_id
 
     def get_ready_hosts(self):
         """Query hosts that are ready to build"""

From 9e7be7a55800e689de35900010dacc8d1078a05c Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 26/92] ...


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 218b276..a29bb0f 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -58,6 +58,7 @@ def getTaskRuns(taskID=None, hostID=None, state=None):
         clauses=clauses, values=locals())
 
     data = query.execute()
+    return data
 
 
 def scheduler_map_task(taskinfo):

From b41b22451b8f15d254c1a92303241b39f1cd1e39 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 27/92] actually pick a host and add a run


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index af567bb..f5b02b1 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -14275,6 +14275,7 @@ class Host(object):
         # for builders using this old api, we fake some of this data to get them to take the
         # task runs assigned to them
         for task in tasks:
+            task['id'] = task['task_id']  # builder expects task id not run id
             task['state'] = koji.TASK_STATES['ASSIGNED']
             task['host_id'] = self.id
         return [[host], tasks]
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index a29bb0f..6cf0b4e 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -2,7 +2,7 @@ import logging
 import psycopg2
 
 import koji
-from .db import QueryProcessor, db_lock
+from .db import QueryProcessor, InsertProcessor, db_lock
 from .util import convert_value
 from koji.context import context
 
@@ -37,6 +37,7 @@ def getTaskRuns(taskID=None, hostID=None, state=None):
         ('task_id', 'task_id'),
         ('host_id', 'host_id'),
         ('host.name', 'host_name'),
+        ('task.method', 'method'),
         ('state', 'state'),
         ("date_part('epoch', create_time)", 'create_ts'),
         ("date_part('epoch', start_time)", 'start_ts'),
@@ -54,7 +55,7 @@ def getTaskRuns(taskID=None, hostID=None, state=None):
 
     query = QueryProcessor(
         columns=fields, aliases=aliases, tables=['scheduler_task_runs'],
-        joins=['LEFT OUTER JOIN host on host_id=host.id'],
+        joins=['host ON host_id=host.id', 'task ON task_id=task.id'],
         clauses=clauses, values=locals())
 
     data = query.execute()
@@ -77,7 +78,9 @@ class TaskScheduler(object):
         self.hosts = None
         self.tasks_by_bin = None
         self.active_tasks = None
+        self.free_tasks = None
         self.maxjobs = 15  # XXX
+        self.capacity_overcommit = 5  # TODO config
 
     def run(self):
         if not db_lock('scheduler', wait=False):
@@ -101,6 +104,11 @@ class TaskScheduler(object):
         self.get_tasks()
         self.get_hosts()
 
+        # debug
+        logger.info(f'Hosts: {len(self.hosts)}')
+        logger.info(f'Free tasks: {len(self.free_tasks)}')
+        logger.info(f'Active tasks: {len(self.active_tasks)}')
+
         # calculate host load and task count
         for task in self.active_tasks:
             # for now, we mirror what kojid updateTasks has been doing
@@ -117,6 +125,7 @@ class TaskScheduler(object):
 
         for host in self.hosts.values():
             host.setdefault('_load', 0.0)
+            host.setdefault('_ntasks', 0)
             # temporary test code
             logger.info(f'Host: {host}')
             ldiff = host['task_load'] - host['_load']
@@ -124,35 +133,36 @@ class TaskScheduler(object):
                 # this is expected in a number of cases, just observing
                 logger.info(f'Host load differs by {ldiff:.2f}: {host}')
 
-        # order bins by available host capacity
-        order = []
-        for _bin in self.hosts_by_bin:
-            hosts = self.hosts_by_bin.get(_bin, [])
-            avail = sum([min(0, h['capacity'] - h['_load']) for h in hosts])
-            order.append((avail, _bin))
-        order.sort()
-
-        # note bin demand for each host
-        for n, (avail, _bin) in enumerate(order):
-            rank = float(n) / len(order)
-            for host in self.hosts_by_bin.get(_bin, []):
-                host.setdefault('_rank', rank)
-                # so host rank is set by the most contentious bin it covers
-                # TODO - we could be smarter here, but it's a start
-
-        # sort binned hosts by rank
-        for _bin in self.hosts_by_bin:
-            hosts = self.hosts_by_bin[_bin]
-            hosts.sort(key=lambda h: h['_rank'], reverse=True)
-            # hosts with least contention first
+        # figure out which hosts *can* take each task
+        # at the moment this is mostly just bin, but in the future it will be more complex
+        for task in self.free_tasks:
+            task['_hosts'] = []
+            min_avail = task['weight'] + self.capacity_overcommit
+            for host in self.hosts_by_bin.get(task['_bin'], []):
+                if (host['capacity'] > host['_load'] and
+                        host['_ntasks'] < self.maxjobs and
+                        host['capacity'] - host['_load'] > min_avail):
+                    task['_hosts'].append(host)
+            logger.info(f'Task {task["task_id"]}: {len(task["_hosts"])} options')
+            for host in task['_hosts']:
+                # demand gives us a rough measure of how much overall load is pending for the host
+                host.setdefault('_demand', 0.0)
+                host['_demand'] += task['weight'] / len(task['_hosts'])
 
         # tasks are already in priority order
         for task in self.free_tasks:
-            hosts = self.hosts_by_bin.get(task['_bin'], [])
-            # these are the hosts that _can_ take this task
-            # TODO - update host ranks as we go
-            # TODO - pick a host and assign
-
+            # pick the host with least demand
+            task['_hosts'].sort(key=lambda h: h['_demand'])
+            min_avail = task['weight'] + self.capacity_overcommit
+            for host in task['_hosts']:
+                if (host['capacity'] > host['_load'] and
+                        host['_ntasks'] < self.maxjobs and
+                        host['capacity'] - host['_load'] > min_avail):
+                    # add run entry
+                    self.add_run(task, host)
+                    # update our totals
+                    host['_load'] += task['weight']
+                    host['_ntasks'] += 1
 
     def get_tasks(self):
         """Get the task data that we need for scheduling"""
@@ -268,6 +278,7 @@ class TaskScheduler(object):
         return hosts
 
     def add_run(self, task, host):
-        insert = InsertProcessor('scheduler_runs')
-        insert.set(task_id=task['id'], host_id=host['id'], state=koji.TASK_STATES['ASSIGNED'])
+        insert = InsertProcessor('scheduler_task_runs')
+        insert.set(task_id=task['task_id'], host_id=host['id'], state=koji.TASK_STATES['ASSIGNED'])
         insert.execute()
+        # TODO actually assign the task entry too

From 74bab34726348df3b9b9028d4a20ba74e7f87f7d Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 28/92] fragment


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 6cf0b4e..f4f77a6 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -27,6 +27,8 @@ def intlist(value):
         return [int(value)]
 
 
+def get_tasks_for_host(hostID):
+
 def getTaskRuns(taskID=None, hostID=None, state=None):
     taskID = convert_value(taskID, cast=int, none_allowed=True)
     hostID = convert_value(hostID, cast=int, none_allowed=True)
@@ -34,11 +36,11 @@ def getTaskRuns(taskID=None, hostID=None, state=None):
 
     fields = (
         ('scheduler_task_runs.id', 'id'),
-        ('task_id', 'task_id'),
-        ('host_id', 'host_id'),
+        ('scheduler_task_runs.task_id', 'task_id'),
+        ('scheduler_task_runs.host_id', 'host_id'),
         ('host.name', 'host_name'),
         ('task.method', 'method'),
-        ('state', 'state'),
+        ('scheduler_task_runs.state', 'state'),
         ("date_part('epoch', create_time)", 'create_ts'),
         ("date_part('epoch', start_time)", 'start_ts'),
         ("date_part('epoch', end_time)", 'end_ts'),

From 4e113a0e1f03f41359169226c49675c8ea916754 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 29/92] update task query for hosts


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index f5b02b1..fc7afcc 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -14266,12 +14266,12 @@ class Host(object):
         relatively complex, so we let the host machines crunch it."""
         host = get_host(self.id)
         host['channels'] = [c['id'] for c in list_channels(hostID=self.id)]
-        tasks = scheduler.getTaskRuns(hostID=self.id)
+        tasks = scheduler.get_tasks_for_host(hostID=self.id)
         if not tasks:
             # try running scheduler
             if scheduler.TaskScheduler().run():
                 # check again
-                tasks = scheduler.getTaskRuns(hostID=self.id)
+                tasks = scheduler.get_tasks_for_host(hostID=self.id)
         # for builders using this old api, we fake some of this data to get them to take the
         # task runs assigned to them
         for task in tasks:
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index f4f77a6..01fa039 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -28,6 +28,29 @@ def intlist(value):
 
 
 def get_tasks_for_host(hostID):
+    """Get the tasks assigned to a given host"""
+    hostID = convert_value(hostID, cast=int, none_allowed=True)
+
+    fields = (
+        ('task.id', 'id'),
+        ('task.state', 'state'),
+        ('task.channel_id', 'channel_id'),
+        ('task.host_id', 'host_id'),
+        ('task.arch', 'arch'),
+        ('task.method', 'method'),
+        ('task.priority', 'priority'),
+        ("date_part('epoch', create_time)", 'create_ts'),
+    )
+    fields, aliases = zip(*fields)
+
+    query = QueryProcessor(
+        columns=fields, aliases=aliases, tables=['task'],
+        clauses=['host_id = %(hostID)s', 'state=%(assigned)s'],
+        values={'hostID': hostID, 'assigned': koji.TASK_STATES['ASSIGNED']},
+    )
+
+    return query.execute()
+
 
 def getTaskRuns(taskID=None, hostID=None, state=None):
     taskID = convert_value(taskID, cast=int, none_allowed=True)

From 81cb2a77138e21c2a8bf1b66c59f8566be9a989d Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 30/92] TEST2


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index fc7afcc..4b1f92b 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -10041,6 +10041,10 @@ class RootExports(object):
     def TEST(self):
         scheduler.TaskScheduler().run()
 
+    def TEST2(self, hostID=1):
+        return Host(hostID).getLoadData()
+        # return scheduler.get_tasks_for_host(hostID)
+
     def restartHosts(self, priority=5, options=None):
         """Spawns restartHosts task
 

From 3bf8b37ebd6967afba14fd5f7ce1cd217c1483b7 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 31/92] do assignment


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 4b1f92b..404389e 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -14276,12 +14276,6 @@ class Host(object):
             if scheduler.TaskScheduler().run():
                 # check again
                 tasks = scheduler.get_tasks_for_host(hostID=self.id)
-        # for builders using this old api, we fake some of this data to get them to take the
-        # task runs assigned to them
-        for task in tasks:
-            task['id'] = task['task_id']  # builder expects task id not run id
-            task['state'] = koji.TASK_STATES['ASSIGNED']
-            task['host_id'] = self.id
         return [[host], tasks]
 
     def isEnabled(self):
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 01fa039..92bd991 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -2,7 +2,7 @@ import logging
 import psycopg2
 
 import koji
-from .db import QueryProcessor, InsertProcessor, db_lock
+from .db import QueryProcessor, InsertProcessor, UpdateProcessor, db_lock
 from .util import convert_value
 from koji.context import context
 
@@ -306,4 +306,10 @@ class TaskScheduler(object):
         insert = InsertProcessor('scheduler_task_runs')
         insert.set(task_id=task['task_id'], host_id=host['id'], state=koji.TASK_STATES['ASSIGNED'])
         insert.execute()
-        # TODO actually assign the task entry too
+        update = UpdateProcessor(
+                'task',
+                data={'host_id': host['id'], 'state': koji.TASK_STATES['ASSIGNED']},
+                clauses=['id=%(task_id)s', 'state=%(free)s'],
+                values={'task_id': task['task_id'], 'free': koji.TASK_STATES['FREE']},
+        )
+        update.execute()

From 21068c0096dc4af2390aedd27cc232552ed43942 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 32/92] typo


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 404389e..3e4c89f 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -262,7 +262,7 @@ class Task(object):
         if state == koji.TASK_STATES['OPEN']:
             # also update run entry
             update2 = UpdateProcessor(
-                'scheduler_runs',
+                'scheduler_task_runs',
                 clauses=['task_id=%(task_id)s', 'host_id=%(host_id)s'],
                 values=locals(),
                 data={'state': state},
@@ -373,7 +373,7 @@ class Task(object):
 
         # also update run entry
         update2 = UpdateProcessor(
-            'scheduler_runs',
+            'scheduler_task_runs',
             clauses=['task_id=%(task_id)s', 'host_id=%(host_id)s'],
             values=locals(),
             data={'state': state},

From 3147c36806ad72a7797a4ab71a0ecf5832d00587 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 33/92] drop some run hooks for now


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 3e4c89f..502a0a6 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -259,16 +259,6 @@ class Task(object):
         if state == koji.TASK_STATES['OPEN']:
             update.rawset(start_time='NOW()')
         update.execute()
-        if state == koji.TASK_STATES['OPEN']:
-            # also update run entry
-            update2 = UpdateProcessor(
-                'scheduler_task_runs',
-                clauses=['task_id=%(task_id)s', 'host_id=%(host_id)s'],
-                values=locals(),
-                data={'state': state},
-                rawdata={'start_time': 'NOW()'},
-            )
-            update2.execute()
         self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES[newstate])
         self.runCallbacks('postTaskStateChange', info, 'host_id', host_id)
         return True
@@ -371,16 +361,6 @@ class Task(object):
                                  rawdata={'completion_time': 'NOW()'})
         update.execute()
 
-        # also update run entry
-        update2 = UpdateProcessor(
-            'scheduler_task_runs',
-            clauses=['task_id=%(task_id)s', 'host_id=%(host_id)s'],
-            values=locals(),
-            data={'state': state},
-            rawdata={'end_time': 'NOW()'},
-        )
-        update2.execute()
-
         self.runCallbacks('postTaskStateChange', info, 'state', state)
         self.runCallbacks('postTaskStateChange', info, 'completion_ts', now)
 

From c7fe0b070e5c95512fd9084db1a5fbba1128dc3d Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 34/92] ignore session times for now, faster channel query


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 92bd991..9e68617 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -266,7 +266,7 @@ class TaskScheduler(object):
             ('host.task_load', 'task_load'),
             ('host_config.arches', 'arches'),
             ('host_config.capacity', 'capacity'),
-            ("date_part('epoch', sessions.update_time)", 'update_ts'),
+        #    ("date_part('epoch', sessions.update_time)", 'update_ts'),
         )
         fields, aliases = zip(*fields)
 
@@ -275,30 +275,33 @@ class TaskScheduler(object):
             columns=fields,
             aliases=aliases,
             clauses=[
-                'host.ready IS TRUE',
+#                'host.ready IS TRUE',
                 'host_config.enabled IS TRUE',
                 'host_config.active IS TRUE',
-                'sessions.expired IS FALSE',
-                'sessions.master IS NULL',
-                "sessions.update_time > NOW() - '5 minutes'::interval"
+#                'sessions.expired IS FALSE',
+#                'sessions.master IS NULL',
+#                "sessions.update_time > NOW() - '5 minutes'::interval"
             ],
             joins=[
-                'sessions USING (user_id)',
+            #    'sessions USING (user_id)',
                 'host_config ON host.id = host_config.host_id'
             ]
         )
 
         hosts = query.execute()
+
+        # also get channel info
+        query = QueryProcessor(
+            tables=['host_channels'],
+            columns=['host_id', 'channel_id'],
+            clauses=['active IS TRUE', 'channels.enabled IS TRUE'],
+            joins=['channels ON host_channels.channel_id = channels.id'],
+        )
+        chan_idx = {}
+        for row in query.execute():
+            chan_idx.setdefault(row['host_id'], []).append(row['channel_id'])
         for host in hosts:
-            query = QueryProcessor(
-                tables=['host_channels'],
-                columns=['channel_id'],
-                clauses=['host_id=%(id)s', 'active IS TRUE', 'enabled IS TRUE'],
-                joins=['channels ON host_channels.channel_id = channels.id'],
-                values=host
-            )
-            rows = query.execute()
-            host['channels'] = [row['channel_id'] for row in rows]
+            host['channels'] = chan_idx.get(host['id'], [])
 
         return hosts
 

From 38c68c1249b06d4759e5b4ddf8d0fbb9e8c0731e Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 35/92] fixes, better ranks, and more logging


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 9e68617..df4f578 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -151,6 +151,7 @@ class TaskScheduler(object):
         for host in self.hosts.values():
             host.setdefault('_load', 0.0)
             host.setdefault('_ntasks', 0)
+            host.setdefault('_demand', 0.0)
             # temporary test code
             logger.info(f'Host: {host}')
             ldiff = host['task_load'] - host['_load']
@@ -162,32 +163,47 @@ class TaskScheduler(object):
         # at the moment this is mostly just bin, but in the future it will be more complex
         for task in self.free_tasks:
             task['_hosts'] = []
-            min_avail = task['weight'] + self.capacity_overcommit
+            min_avail = min(0, task['weight'] - self.capacity_overcommit)
             for host in self.hosts_by_bin.get(task['_bin'], []):
-                if (host['capacity'] > host['_load'] and
-                        host['_ntasks'] < self.maxjobs and
-                        host['capacity'] - host['_load'] > min_avail):
+                if (host['capacity'] - host['_load'] > min_avail and
+                        host['_ntasks'] < self.maxjobs):
                     task['_hosts'].append(host)
             logger.info(f'Task {task["task_id"]}: {len(task["_hosts"])} options')
+            #import pdb; pdb.set_trace()
             for host in task['_hosts']:
                 # demand gives us a rough measure of how much overall load is pending for the host
                 host.setdefault('_demand', 0.0)
                 host['_demand'] += task['weight'] / len(task['_hosts'])
 
+        # normalize demand to 1
+        max_demand = sum([h['_demand'] for h in self.hosts.values()])
+        if max_demand > 0.0:
+            for h in self.hosts.values():
+                h['_demand'] = (h['_demand'] / max_demand)
+
+        for h in self.hosts.values():
+            self._rank_host(h)
+
         # tasks are already in priority order
         for task in self.free_tasks:
-            # pick the host with least demand
-            task['_hosts'].sort(key=lambda h: h['_demand'])
-            min_avail = task['weight'] + self.capacity_overcommit
+            min_avail = task['weight'] - self.capacity_overcommit
+            task['_hosts'].sort(key=lambda h: h['_rank'])
+            logger.debug('Task %i choices: %s', task['task_id'], [(h['name'], "%(_rank).2f" %h) for h in task['_hosts']])
             for host in task['_hosts']:
-                if (host['capacity'] > host['_load'] and
-                        host['_ntasks'] < self.maxjobs and
-                        host['capacity'] - host['_load'] > min_avail):
+                if (host['capacity'] - host['_load'] > min_avail and
+                        host['_ntasks'] < self.maxjobs):
                     # add run entry
                     self.add_run(task, host)
-                    # update our totals
+                    # update our totals and rank
                     host['_load'] += task['weight']
                     host['_ntasks'] += 1
+                    self._rank_host(host)
+                    break
+            else:
+                logger.debug('Could not assign task %s', task['task_id'])
+
+    def _rank_host(self, host):
+        host['_rank'] = host['_load'] + host['_ntasks'] + host['_demand']
 
     def get_tasks(self):
         """Get the task data that we need for scheduling"""
@@ -306,6 +322,8 @@ class TaskScheduler(object):
         return hosts
 
     def add_run(self, task, host):
+        logger.info('Assigning task %s (%s) to host %s',
+                    task['task_id'], task['method'], host['name'])
         insert = InsertProcessor('scheduler_task_runs')
         insert.set(task_id=task['task_id'], host_id=host['id'], state=koji.TASK_STATES['ASSIGNED'])
         insert.execute()

From de90b1350a75eb776232bf3430d47a392303de49 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 36/92] check on tasks, partially implement assign timeout


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 502a0a6..016bb11 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -10018,8 +10018,10 @@ def _delete_event_id():
 class RootExports(object):
     '''Contains functions that are made available via XMLRPC'''
 
-    def TEST(self):
+    def TEST(self, fail=False):
         scheduler.TaskScheduler().run()
+        if fail:
+            raise Exception('DEBUG')
 
     def TEST2(self, hostID=1):
         return Host(hostID).getLoadData()
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index df4f578..ea98229 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -1,5 +1,6 @@
 import logging
 import psycopg2
+import time
 
 import koji
 from .db import QueryProcessor, InsertProcessor, UpdateProcessor, db_lock
@@ -52,21 +53,19 @@ def get_tasks_for_host(hostID):
     return query.execute()
 
 
-def getTaskRuns(taskID=None, hostID=None, state=None):
+def getTaskRuns(taskID=None, hostID=None, active=None):
     taskID = convert_value(taskID, cast=int, none_allowed=True)
     hostID = convert_value(hostID, cast=int, none_allowed=True)
-    state = convert_value(state, cast=intlist, none_allowed=True)
+    active = convert_value(active, cast=bool, none_allowed=True)
 
     fields = (
         ('scheduler_task_runs.id', 'id'),
         ('scheduler_task_runs.task_id', 'task_id'),
         ('scheduler_task_runs.host_id', 'host_id'),
-        ('host.name', 'host_name'),
-        ('task.method', 'method'),
-        ('scheduler_task_runs.state', 'state'),
-        ("date_part('epoch', create_time)", 'create_ts'),
-        ("date_part('epoch', start_time)", 'start_ts'),
-        ("date_part('epoch', end_time)", 'end_ts'),
+    #    ('host.name', 'host_name'),
+    #    ('task.method', 'method'),
+        ('scheduler_task_runs.active', 'active'),
+        ("date_part('epoch', scheduler_task_runs.create_time)", 'create_ts'),
     )
     fields, aliases = zip(*fields)
 
@@ -75,16 +74,15 @@ def getTaskRuns(taskID=None, hostID=None, state=None):
         clauses.append('task_id = %(taskID)s')
     if hostID is not None:
         clauses.append('host_id = %(hostID)s')
-    if state is not None:
-        clauses.append('host_id IN %(state)s')
+    if active is not None:
+        clauses.append('active = %(active)s')
 
     query = QueryProcessor(
         columns=fields, aliases=aliases, tables=['scheduler_task_runs'],
-        joins=['host ON host_id=host.id', 'task ON task_id=task.id'],
+    #    joins=['host ON host_id=host.id', 'task ON task_id=task.id'],
         clauses=clauses, values=locals())
 
-    data = query.execute()
-    return data
+    return query.execute()
 
 
 def scheduler_map_task(taskinfo):
@@ -104,8 +102,11 @@ class TaskScheduler(object):
         self.tasks_by_bin = None
         self.active_tasks = None
         self.free_tasks = None
+
+        # TODO these things need proper config
         self.maxjobs = 15  # XXX
-        self.capacity_overcommit = 5  # TODO config
+        self.capacity_overcommit = 5
+        self.assign_timeout = 300
 
     def run(self):
         if not db_lock('scheduler', wait=False):
@@ -114,22 +115,16 @@ class TaskScheduler(object):
 
         self.do_schedule()
         # TODO clean up bad data (e.g. active tasks with no host)
-        # TODO check for runs that aren't getting picked up
+        self.check_active_tasks()
 
         return True
 
-    def get_runs(self):
-        runs = getTaskRuns()
-        runs_by_task = {}
-        for run in runs:
-            runs_by_task.setdefault(run['task_id'], [])
-            runs_by_task[run['task_id']].append(run)
-
     def do_schedule(self):
         self.get_tasks()
         self.get_hosts()
 
         # debug
+        logger.info('Running task scheduler')
         logger.info(f'Hosts: {len(self.hosts)}')
         logger.info(f'Free tasks: {len(self.free_tasks)}')
         logger.info(f'Active tasks: {len(self.active_tasks)}')
@@ -205,6 +200,52 @@ class TaskScheduler(object):
     def _rank_host(self, host):
         host['_rank'] = host['_load'] + host['_ntasks'] + host['_demand']
 
+    def check_active_tasks(self):
+        """Check on active tasks"""
+        runs = self.get_active_runs()
+        logger.info('Found %i active runs', len(runs))
+        logger.info('Checking on %i active tasks', len(self.active_tasks))
+        for task in self.active_tasks:
+            if task['state'] == koji.TASK_STATES['ASSIGNED']:
+                # TODO check time since assigned
+                # if not taken within a timeout
+                #  - if host not checking in, then make sure host marked unavail and free
+                #  - if host *is* checking in, then treat as refusal and free
+                taskruns = runs.get(task['task_id'], [])
+                if not taskruns:
+                    logger.error('No active run for assigned task %(task_id)s', task)
+                    # TODO free
+                    continue
+                else:
+                    if len(taskruns) > 1:
+                        logger.error('Multiple active run entries for assigned task %(task_id)s',
+                                     task)
+                        # TODO fix
+                    age = time.time() - min([r['create_ts'] for r in taskruns])
+                    if age > self.assign_timeout:
+                        # TODO free
+                        # TODO check host too
+                        logger.info('Task assignment timeout for %(task_id)s', task)
+                        pass
+            elif task['state'] == koji.TASK_STATES['OPEN']:
+                # TODO sanity check host
+                if not task['host_id']:
+                    # shouldn't happen
+                    # TODO
+                    continue
+                host = self.hosts.get(task['host_id'])
+                if not host:
+                    logger.error('Host for task is not available')
+
+    def get_active_runs(self):
+        runs = getTaskRuns(active=True)
+        runs_by_task = {}
+        for run in runs:
+            runs_by_task.setdefault(run['task_id'], [])
+            runs_by_task[run['task_id']].append(run)
+
+        return runs_by_task
+
     def get_tasks(self):
         """Get the task data that we need for scheduling"""
 
@@ -324,9 +365,22 @@ class TaskScheduler(object):
     def add_run(self, task, host):
         logger.info('Assigning task %s (%s) to host %s',
                     task['task_id'], task['method'], host['name'])
+
+        # mark any older runs inactive
+        update = UpdateProcessor(
+                'scheduler_task_runs',
+                data={'active': False},
+                clauses=['task_id=%(task_id)s', 'active = TRUE'],
+                values={'task_id': task['task_id']},
+        )
+        update.execute()
+
+        # add the new run
         insert = InsertProcessor('scheduler_task_runs')
-        insert.set(task_id=task['task_id'], host_id=host['id'], state=koji.TASK_STATES['ASSIGNED'])
+        insert.set(task_id=task['task_id'], host_id=host['id'])
         insert.execute()
+
+        # mark the task assigned
         update = UpdateProcessor(
                 'task',
                 data={'host_id': host['id'], 'state': koji.TASK_STATES['ASSIGNED']},
diff --git a/schemas/schema.sql b/schemas/schema.sql
index f97ac2e..57ff6f5 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -989,14 +989,11 @@ CREATE TABLE scheduler_task_runs (
         id SERIAL NOT NULL PRIMARY KEY,
         task_id INTEGER REFERENCES task (id) NOT NULL,
         host_id INTEGER REFERENCES host (id) NOT NULL,
-        state INTEGER NOT NULL,
-        create_time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-        start_time TIMESTAMPTZ,
-        end_time TIMESTAMPTZ
+        active BOOLEAN NOT NULL DEFAULT TRUE,
+        create_time TIMESTAMPTZ NOT NULL DEFAULT NOW()
 ) WITHOUT OIDS;
 CREATE INDEX scheduler_task_runs_task ON scheduler_task_runs(task_id);
 CREATE INDEX scheduler_task_runs_host ON scheduler_task_runs(host_id);
-CREATE INDEX scheduler_task_runs_state ON scheduler_task_runs(state);
 CREATE INDEX scheduler_task_runs_create_time ON scheduler_task_runs(create_time);
 
 

From dbce44216006382dfc29917881ccd2896096a36c Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 37/92] move Task class


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 016bb11..7322834 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -112,460 +112,6 @@ def xform_user_krb(entry):
     return entry
 
 
-class Task(object):
-    """A task for the build hosts"""
-
-    fields = (
-        ('task.id', 'id'),
-        ('task.state', 'state'),
-        ('task.create_time', 'create_time'),
-        ("date_part('epoch', create_time)", 'create_ts'),
-        ('task.start_time', 'start_time'),
-        ("date_part('epoch', task.start_time)", 'start_ts'),
-        ('task.completion_time', 'completion_time'),
-        ("date_part('epoch', completion_time)", 'completion_ts'),
-        ('task.channel_id', 'channel_id'),
-        ('task.host_id', 'host_id'),
-        ('task.parent', 'parent'),
-        ('task.label', 'label'),
-        ('task.waiting', 'waiting'),
-        ('task.awaited', 'awaited'),
-        ('task.owner', 'owner'),
-        ('task.method', 'method'),
-        ('task.arch', 'arch'),
-        ('task.priority', 'priority'),
-        ('task.weight', 'weight'))
-
-    def __init__(self, id):
-        self.id = convert_value(id, cast=int)
-        self.logger = logging.getLogger("koji.hub.Task")
-
-    def _split_fields(self, fields=None):
-        """Helper function for split fields to QueryProcessor's
-       columns/aliases options"""
-        if fields is None:
-            fields = self.fields
-        columns = [f[0] for f in fields]
-        aliases = [f[1] for f in fields]
-        return columns, aliases
-
-    def verifyHost(self, host_id=None):
-        """Verify that host owns task"""
-        if host_id is None:
-            host_id = context.session.host_id
-        if host_id is None:
-            return False
-        task_id = self.id
-        # getting a row lock on this task to ensure task assignment sanity
-        # no other concurrent transaction should be altering this row
-        query = QueryProcessor(tables=['task'], columns=['state', 'host_id'],
-                               clauses=['id=%(task_id)s'], values={'task_id': task_id},
-                               opts={'rowlock': True})
-        r = query.executeOne()
-        if not r:
-            raise koji.GenericError("No such task: %i" % task_id)
-        return (r['state'] == koji.TASK_STATES['OPEN'] and r['host_id'] == host_id)
-
-    def assertHost(self, host_id):
-        if not self.verifyHost(host_id):
-            raise koji.ActionNotAllowed("host %d does not own task %d" % (host_id, self.id))
-
-    def getOwner(self):
-        """Return the owner (user_id) for this task"""
-        query = QueryProcessor(tables=['task'], columns=['owner'],
-                               clauses=['id=%(id)i'], values=vars(self))
-        return query.singleValue()
-
-    def verifyOwner(self, user_id=None):
-        """Verify that user owns task"""
-        if user_id is None:
-            user_id = context.session.user_id
-        if user_id is None:
-            return False
-        task_id = self.id
-        # getting a row lock on this task to ensure task state sanity
-        query = QueryProcessor(tables=['task'], columns=['owner'],
-                               clauses=['id=%(task_id)s'], values={'task_id': task_id},
-                               opts={'rowlock': True})
-        owner = query.singleValue(strict=False)
-        if not owner:
-            raise koji.GenericError("No such task: %i" % task_id)
-        return (owner == user_id)
-
-    def assertOwner(self, user_id=None):
-        if not self.verifyOwner(user_id):
-            raise koji.ActionNotAllowed("user %d does not own task %d" % (user_id, self.id))
-
-    def lock(self, host_id, newstate='OPEN', force=False):
-        """Attempt to associate the task for host, either to assign or open
-
-        returns True if successful, False otherwise"""
-        info = self.getInfo(request=True)
-        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES[newstate])
-        self.runCallbacks('preTaskStateChange', info, 'host_id', host_id)
-        # we use row-level locks to keep things sane
-        # note the QueryProcessor...opts={'rowlock': True}
-        task_id = self.id
-        if not force:
-            query = QueryProcessor(columns=['state', 'host_id'], tables=['task'],
-                                   clauses=['id=%(task_id)s'], values={'task_id': task_id},
-                                   opts={'rowlock': True})
-            r = query.executeOne()
-            if not r:
-                raise koji.GenericError("No such task: %i" % task_id)
-            state = r['state']
-            otherhost = r['host_id']
-            if state == koji.TASK_STATES['FREE']:
-                if otherhost is not None:
-                    log_error(f"Error: task {task_id} is both free "
-                              f"and handled by host {otherhost}")
-                    return False
-            elif state == koji.TASK_STATES['ASSIGNED']:
-                if otherhost is None:
-                    log_error(f"Error: task {task_id} is assigned, but no host is really assigned")
-                    return False
-                elif otherhost != host_id:
-                    # task is assigned to someone else, no error just return
-                    return False
-                elif newstate == 'ASSIGNED':
-                    # double assign is a weird situation but we can return True as state doesn't
-                    # really change
-                    log_error(f"Error: double assign of task {task_id} and host {host_id}")
-                    return True
-                # otherwise the task is assigned to host_id, so keep going
-            elif state == koji.TASK_STATES['CANCELED']:
-                # it is ok that task was canceled meanwhile
-                return False
-            elif state == koji.TASK_STATES['OPEN']:
-                if otherhost is None:
-                    log_error(f"Error: task {task_id} is opened but not handled by any host")
-                elif otherhost == host_id:
-                    log_error(f"Error: task {task_id} is already open and handled by "
-                              f"{host_id} (double open/assign)")
-                return False
-            else:
-                # state is CLOSED or FAILED
-                if otherhost is None:
-                    log_error(f"Error: task {task_id} is non-free but not handled by any host "
-                              f"(state {koji.TASK_STATES[state]})")
-                return False
-        # if we reach here, task is either
-        #  - free and unlocked
-        #  - assigned to host_id
-        #  - force option is enabled
-        state = koji.TASK_STATES[newstate]
-        update = UpdateProcessor('task', clauses=['id=%(task_id)i'], values=locals())
-        update.set(state=state, host_id=host_id)
-        if state == koji.TASK_STATES['OPEN']:
-            update.rawset(start_time='NOW()')
-        update.execute()
-        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES[newstate])
-        self.runCallbacks('postTaskStateChange', info, 'host_id', host_id)
-        return True
-
-    def assign(self, host_id, force=False):
-        """Attempt to assign the task to host.
-
-        returns True if successful, False otherwise"""
-        return self.lock(host_id, 'ASSIGNED', force)
-
-    def open(self, host_id):
-        """Attempt to open the task for host.
-
-        returns task data if successful, None otherwise"""
-        if self.lock(host_id, 'OPEN'):
-            # get more complete data to return
-            fields = self.fields + (('task.request', 'request'),)
-            query = QueryProcessor(tables=['task'], clauses=['id=%(id)i'], values=vars(self),
-                                   columns=[f[0] for f in fields], aliases=[f[1] for f in fields])
-            ret = query.executeOne()
-            if ret['request'].find('<?xml', 0, 10) == -1:
-                # handle older base64 encoded data
-                data = base64.b64decode(ret['request'])
-                # we can't return raw bytes and this /should/ be a valid string
-                ret['request'] = decode_bytes(data)
-            return ret
-        else:
-            return None
-
-    def free(self):
-        """Free a task"""
-        info = self.getInfo(request=True)
-        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES['FREE'])
-        self.runCallbacks('preTaskStateChange', info, 'host_id', None)
-        # access checks should be performed by calling function
-        query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(id)i'],
-                               values=vars(self), opts={'rowlock': True})
-        oldstate = query.singleValue(strict=False)
-        if not oldstate:
-            raise koji.GenericError("No such task: %i" % self.id)
-        if koji.TASK_STATES[oldstate] in ['CLOSED', 'CANCELED', 'FAILED']:
-            raise koji.GenericError("Cannot free task %i, state is %s" %
-                                    (self.id, koji.TASK_STATES[oldstate]))
-        newstate = koji.TASK_STATES['FREE']
-        newhost = None
-        update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
-                                 data={'state': newstate, 'host_id': newhost})
-        update.execute()
-        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES['FREE'])
-        self.runCallbacks('postTaskStateChange', info, 'host_id', None)
-        return True
-
-    def setWeight(self, weight):
-        """Set weight for task"""
-        weight = convert_value(weight, cast=float)
-        info = self.getInfo(request=True)
-        self.runCallbacks('preTaskStateChange', info, 'weight', weight)
-        # access checks should be performed by calling function
-        update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
-                                 data={'weight': weight})
-        update.execute()
-        self.runCallbacks('postTaskStateChange', info, 'weight', weight)
-
-    def setPriority(self, priority, recurse=False):
-        """Set priority for task"""
-        priority = convert_value(priority, cast=int)
-        info = self.getInfo(request=True)
-        self.runCallbacks('preTaskStateChange', info, 'priority', priority)
-        # access checks should be performed by calling function
-        update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
-                                 data={'priority': priority})
-        update.execute()
-        self.runCallbacks('postTaskStateChange', info, 'priority', priority)
-
-        if recurse:
-            # Change priority of child tasks
-            query = QueryProcessor(tables=['task'], columns=['id'],
-                                   clauses=['parent = %(task_id)s'],
-                                   values={'task_id': self.id},
-                                   opts={'asList': True})
-            for (child_id,) in query.execute():
-                Task(child_id).setPriority(priority, recurse=True)
-
-    def _close(self, result, state):
-        """Mark task closed and set response
-
-        Returns True if successful, False if not"""
-        # access checks should be performed by calling function
-        # this is an approximation, and will be different than what is in the database
-        # the actual value should be retrieved from the 'new' value of the post callback
-        now = time.time()
-        info = self.getInfo(request=True)
-        info['result'] = result
-        self.runCallbacks('preTaskStateChange', info, 'state', state)
-        self.runCallbacks('preTaskStateChange', info, 'completion_ts', now)
-        # get the result from the info dict, so callbacks have a chance to modify it
-        update = UpdateProcessor('task', clauses=['id = %(task_id)d'],
-                                 values={'task_id': self.id},
-                                 data={'result': info['result'], 'state': state},
-                                 rawdata={'completion_time': 'NOW()'})
-        update.execute()
-
-        self.runCallbacks('postTaskStateChange', info, 'state', state)
-        self.runCallbacks('postTaskStateChange', info, 'completion_ts', now)
-
-    def close(self, result):
-        # access checks should be performed by calling function
-        self._close(result, koji.TASK_STATES['CLOSED'])
-
-    def fail(self, result):
-        # access checks should be performed by calling function
-        self._close(result, koji.TASK_STATES['FAILED'])
-
-    def getState(self):
-        query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(id)i'],
-                               values=vars(self))
-        return query.singleValue()
-
-    def isFinished(self):
-        return (koji.TASK_STATES[self.getState()] in ['CLOSED', 'CANCELED', 'FAILED'])
-
-    def isCanceled(self):
-        return (self.getState() == koji.TASK_STATES['CANCELED'])
-
-    def isFailed(self):
-        return (self.getState() == koji.TASK_STATES['FAILED'])
-
-    def cancel(self, recurse=True):
-        """Cancel this task.
-
-        A task can only be canceled if it is not already in the 'CLOSED' state.
-        If it is, no action will be taken.  Return True if the task is
-        successfully canceled, or if it was already canceled, False if it is
-        closed."""
-        # access checks should be performed by calling function
-        now = time.time()
-        info = self.getInfo(request=True)
-        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES['CANCELED'])
-        self.runCallbacks('preTaskStateChange', info, 'completion_ts', now)
-        query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(task_id)s'],
-                               values={'task_id': self.id}, opts={'rowlock': True})
-        state = query.singleValue()
-        st_canceled = koji.TASK_STATES['CANCELED']
-        st_closed = koji.TASK_STATES['CLOSED']
-        st_failed = koji.TASK_STATES['FAILED']
-        if state == st_canceled:
-            return True
-        elif state in [st_closed, st_failed]:
-            return False
-        update = UpdateProcessor('task', clauses=['id = %(task_id)i'], values={'task_id': self.id},
-                                 data={'state': st_canceled}, rawdata={'completion_time': 'NOW()'})
-        update.execute()
-        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES['CANCELED'])
-        self.runCallbacks('postTaskStateChange', info, 'completion_ts', now)
-        # cancel associated builds (only if state is 'BUILDING')
-        # since we check build state, we avoid loops with cancel_build on our end
-        b_building = koji.BUILD_STATES['BUILDING']
-        query = QueryProcessor(tables=['build'], columns=['id'],
-                               clauses=['task_id = %(task_id)i', 'state = %(b_building)i'],
-                               values={'task_id': self.id, 'b_building': b_building},
-                               opts={'rowlock': True, 'asList': True})
-        for (build_id,) in query.execute():
-            cancel_build(build_id, cancel_task=False)
-        if recurse:
-            # also cancel child tasks
-            self.cancelChildren()
-        return True
-
-    def cancelChildren(self):
-        """Cancel child tasks"""
-        query = QueryProcessor(tables=['task'], columns=['id'], clauses=['parent = %(task_id)i'],
-                               values={'task_id': self.id}, opts={'asList': True})
-        for (id,) in query.execute():
-            Task(id).cancel(recurse=True)
-
-    def cancelFull(self, strict=True):
-        """Cancel this task and every other task in its group
-
-        If strict is true, then this must be a top-level task
-        Otherwise we will follow up the chain to find the top-level task
-        """
-        task_id = self.id
-        query = QueryProcessor(tables=['task'], columns=['parent'],
-                               clauses=['id = %(task_id)i'],
-                               values={'task_id': task_id}, opts={'rowlock': True})
-        parent = query.singleValue(strict=False)
-        if parent is not None:
-            if strict:
-                raise koji.GenericError("Task %d is not top-level (parent=%d)" % (task_id, parent))
-            # otherwise, find the top-level task and go from there
-            seen = {task_id: 1}
-            while parent is not None:
-                if parent in seen:
-                    raise koji.GenericError("Task LOOP at task %i" % task_id)
-                task_id = parent
-                seen[task_id] = 1
-                query.values = {'task_id': task_id}
-                parent = query.singleValue()
-            return Task(task_id).cancelFull(strict=True)
-        # We handle the recursion ourselves, since self.cancel will stop at
-        # canceled or closed tasks.
-        tasklist = [task_id]
-        seen = {}
-        # query for use in loop
-        for task_id in tasklist:
-            if task_id in seen:
-                # shouldn't happen
-                raise koji.GenericError("Task LOOP at task %i" % task_id)
-            seen[task_id] = 1
-            Task(task_id).cancel(recurse=False)
-            query = QueryProcessor(tables=['task'], columns=['id'],
-                                   clauses=['parent = %(task_id)i'],
-                                   values={'task_id': task_id}, opts={'asList': True})
-            for (child_id,) in query.execute():
-                tasklist.append(child_id)
-
-    def getRequest(self):
-        query = QueryProcessor(columns=['request'], tables=['task'],
-                               clauses=['id = %(id)i'], values={'id': self.id})
-        xml_request = query.singleValue()
-        if xml_request.find('<?xml', 0, 10) == -1:
-            # handle older base64 encoded data
-            xml_request = base64.b64decode(xml_request)
-        # note: loads accepts either bytes or string
-        params, method = xmlrpc.client.loads(xml_request)
-        return params
-
-    def getResult(self, raise_fault=True):
-        query = QueryProcessor(tables=['task'], columns=['state', 'result'],
-                               clauses=['id = %(id)i'], values={'id': self.id})
-        r = query.executeOne()
-        if not r:
-            raise koji.GenericError("No such task")
-        state = r['state']
-        xml_result = r['result']
-        if koji.TASK_STATES[state] == 'CANCELED':
-            raise koji.GenericError("Task %i is canceled" % self.id)
-        elif koji.TASK_STATES[state] not in ['CLOSED', 'FAILED']:
-            raise koji.GenericError("Task %i is not finished" % self.id)
-        if xml_result.find('<?xml', 0, 10) == -1:
-            # handle older base64 encoded data
-            xml_result = base64.b64decode(xml_result)
-        try:
-            # If the result is a Fault, then loads will raise it
-            # This is normally what we want to happen
-            result, method = xmlrpc.client.loads(xml_result)
-        except xmlrpc.client.Fault as fault:
-            if raise_fault:
-                raise
-            # Note that you can't really return a fault over xmlrpc, except by
-            # raising it. We return a dictionary in the same format that
-            # multiCall does.
-            return {'faultCode': fault.faultCode, 'faultString': fault.faultString}
-        return result[0]
-
-    def getInfo(self, strict=True, request=False):
-        """Return information about the task in a dictionary.  If "request" is True,
-        the request will be decoded and included in the dictionary."""
-        columns, aliases = self._split_fields()
-        query = QueryProcessor(columns=columns, aliases=aliases,
-                               tables=['task'], clauses=['id = %(id)i'],
-                               values={'id': self.id})
-        result = query.executeOne(strict=strict)
-        if result and request:
-            result['request'] = self.getRequest()
-        return result
-
-    def getChildren(self, request=False):
-        """Return information about tasks with this task as their
-        parent.  If there are no such Tasks, return an empty list."""
-        fields = self.fields
-        if request:
-            fields = fields + (('request', 'request'),)
-        columns, aliases = self._split_fields(fields)
-        query = QueryProcessor(columns=columns, aliases=aliases,
-                               tables=['task'], clauses=['parent = %(id)i'],
-                               values={'id': self.id})
-        results = query.execute()
-        if request:
-            for task in results:
-                if task['request'].find('<?xml', 0, 10) == -1:
-                    # handle older base64 encoded data
-                    task['request'] = base64.b64decode(task['request'])
-                # note: loads accepts either bytes or string
-                task['request'] = xmlrpc.client.loads(task['request'])[0]
-        return results
-
-    def runCallbacks(self, cbtype, old_info, attr, new_val):
-        if cbtype.startswith('pre'):
-            info = old_info
-        elif cbtype.startswith('post'):
-            info = self.getInfo(request=True)
-            if info['state'] == koji.TASK_STATES['CLOSED']:
-                # if task is closed, include the result as well
-                info['result'] = self.getResult()
-            new_val = info[attr]
-        else:
-            raise koji.GenericError('No such callback type: %s' % cbtype)
-        old_val = old_info[attr]
-        if attr == 'state':
-            # state is passed in as an integer, but we want to use the string
-            old_val = koji.TASK_STATES[old_val]
-            new_val = koji.TASK_STATES[new_val]
-        koji.plugin.run_callbacks(cbtype, attribute=attr, old=old_val, new=new_val,
-                                  info=info)
-
-
 def make_task(method, arglist, **opts):
     """Create a task
 
diff --git a/kojihub/task.py b/kojihub/task.py
new file mode 100644
index 0000000..e515b38
--- /dev/null
+++ b/kojihub/task.py
@@ -0,0 +1,455 @@
+# Task related hub code
+
+
+class Task(object):
+    """A task for the build hosts"""
+
+    fields = (
+        ('task.id', 'id'),
+        ('task.state', 'state'),
+        ('task.create_time', 'create_time'),
+        ("date_part('epoch', create_time)", 'create_ts'),
+        ('task.start_time', 'start_time'),
+        ("date_part('epoch', task.start_time)", 'start_ts'),
+        ('task.completion_time', 'completion_time'),
+        ("date_part('epoch', completion_time)", 'completion_ts'),
+        ('task.channel_id', 'channel_id'),
+        ('task.host_id', 'host_id'),
+        ('task.parent', 'parent'),
+        ('task.label', 'label'),
+        ('task.waiting', 'waiting'),
+        ('task.awaited', 'awaited'),
+        ('task.owner', 'owner'),
+        ('task.method', 'method'),
+        ('task.arch', 'arch'),
+        ('task.priority', 'priority'),
+        ('task.weight', 'weight'))
+
+    def __init__(self, id):
+        self.id = convert_value(id, cast=int)
+        self.logger = logging.getLogger("koji.hub.Task")
+
+    def _split_fields(self, fields=None):
+        """Helper function for split fields to QueryProcessor's
+       columns/aliases options"""
+        if fields is None:
+            fields = self.fields
+        columns = [f[0] for f in fields]
+        aliases = [f[1] for f in fields]
+        return columns, aliases
+
+    def verifyHost(self, host_id=None):
+        """Verify that host owns task"""
+        if host_id is None:
+            host_id = context.session.host_id
+        if host_id is None:
+            return False
+        task_id = self.id
+        # getting a row lock on this task to ensure task assignment sanity
+        # no other concurrent transaction should be altering this row
+        query = QueryProcessor(tables=['task'], columns=['state', 'host_id'],
+                               clauses=['id=%(task_id)s'], values={'task_id': task_id},
+                               opts={'rowlock': True})
+        r = query.executeOne()
+        if not r:
+            raise koji.GenericError("No such task: %i" % task_id)
+        return (r['state'] == koji.TASK_STATES['OPEN'] and r['host_id'] == host_id)
+
+    def assertHost(self, host_id):
+        if not self.verifyHost(host_id):
+            raise koji.ActionNotAllowed("host %d does not own task %d" % (host_id, self.id))
+
+    def getOwner(self):
+        """Return the owner (user_id) for this task"""
+        query = QueryProcessor(tables=['task'], columns=['owner'],
+                               clauses=['id=%(id)i'], values=vars(self))
+        return query.singleValue()
+
+    def verifyOwner(self, user_id=None):
+        """Verify that user owns task"""
+        if user_id is None:
+            user_id = context.session.user_id
+        if user_id is None:
+            return False
+        task_id = self.id
+        # getting a row lock on this task to ensure task state sanity
+        query = QueryProcessor(tables=['task'], columns=['owner'],
+                               clauses=['id=%(task_id)s'], values={'task_id': task_id},
+                               opts={'rowlock': True})
+        owner = query.singleValue(strict=False)
+        if not owner:
+            raise koji.GenericError("No such task: %i" % task_id)
+        return (owner == user_id)
+
+    def assertOwner(self, user_id=None):
+        if not self.verifyOwner(user_id):
+            raise koji.ActionNotAllowed("user %d does not own task %d" % (user_id, self.id))
+
+    def lock(self, host_id, newstate='OPEN', force=False):
+        """Attempt to associate the task for host, either to assign or open
+
+        returns True if successful, False otherwise"""
+        info = self.getInfo(request=True)
+        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES[newstate])
+        self.runCallbacks('preTaskStateChange', info, 'host_id', host_id)
+        # we use row-level locks to keep things sane
+        # note the QueryProcessor...opts={'rowlock': True}
+        task_id = self.id
+        if not force:
+            query = QueryProcessor(columns=['state', 'host_id'], tables=['task'],
+                                   clauses=['id=%(task_id)s'], values={'task_id': task_id},
+                                   opts={'rowlock': True})
+            r = query.executeOne()
+            if not r:
+                raise koji.GenericError("No such task: %i" % task_id)
+            state = r['state']
+            otherhost = r['host_id']
+            if state == koji.TASK_STATES['FREE']:
+                if otherhost is not None:
+                    log_error(f"Error: task {task_id} is both free "
+                              f"and handled by host {otherhost}")
+                    return False
+            elif state == koji.TASK_STATES['ASSIGNED']:
+                if otherhost is None:
+                    log_error(f"Error: task {task_id} is assigned, but no host is really assigned")
+                    return False
+                elif otherhost != host_id:
+                    # task is assigned to someone else, no error just return
+                    return False
+                elif newstate == 'ASSIGNED':
+                    # double assign is a weird situation but we can return True as state doesn't
+                    # really change
+                    log_error(f"Error: double assign of task {task_id} and host {host_id}")
+                    return True
+                # otherwise the task is assigned to host_id, so keep going
+            elif state == koji.TASK_STATES['CANCELED']:
+                # it is ok that task was canceled meanwhile
+                return False
+            elif state == koji.TASK_STATES['OPEN']:
+                if otherhost is None:
+                    log_error(f"Error: task {task_id} is opened but not handled by any host")
+                elif otherhost == host_id:
+                    log_error(f"Error: task {task_id} is already open and handled by "
+                              f"{host_id} (double open/assign)")
+                return False
+            else:
+                # state is CLOSED or FAILED
+                if otherhost is None:
+                    log_error(f"Error: task {task_id} is non-free but not handled by any host "
+                              f"(state {koji.TASK_STATES[state]})")
+                return False
+        # if we reach here, task is either
+        #  - free and unlocked
+        #  - assigned to host_id
+        #  - force option is enabled
+        state = koji.TASK_STATES[newstate]
+        update = UpdateProcessor('task', clauses=['id=%(task_id)i'], values=locals())
+        update.set(state=state, host_id=host_id)
+        if state == koji.TASK_STATES['OPEN']:
+            update.rawset(start_time='NOW()')
+        update.execute()
+        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES[newstate])
+        self.runCallbacks('postTaskStateChange', info, 'host_id', host_id)
+        return True
+
+    def assign(self, host_id, force=False):
+        """Attempt to assign the task to host.
+
+        returns True if successful, False otherwise"""
+        return self.lock(host_id, 'ASSIGNED', force)
+
+    def open(self, host_id):
+        """Attempt to open the task for host.
+
+        returns task data if successful, None otherwise"""
+        if self.lock(host_id, 'OPEN'):
+            # get more complete data to return
+            fields = self.fields + (('task.request', 'request'),)
+            query = QueryProcessor(tables=['task'], clauses=['id=%(id)i'], values=vars(self),
+                                   columns=[f[0] for f in fields], aliases=[f[1] for f in fields])
+            ret = query.executeOne()
+            if ret['request'].find('<?xml', 0, 10) == -1:
+                # handle older base64 encoded data
+                data = base64.b64decode(ret['request'])
+                # we can't return raw bytes and this /should/ be a valid string
+                ret['request'] = decode_bytes(data)
+            return ret
+        else:
+            return None
+
+    def free(self):
+        """Free a task"""
+        info = self.getInfo(request=True)
+        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES['FREE'])
+        self.runCallbacks('preTaskStateChange', info, 'host_id', None)
+        # access checks should be performed by calling function
+        query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(id)i'],
+                               values=vars(self), opts={'rowlock': True})
+        oldstate = query.singleValue(strict=False)
+        if not oldstate:
+            raise koji.GenericError("No such task: %i" % self.id)
+        if koji.TASK_STATES[oldstate] in ['CLOSED', 'CANCELED', 'FAILED']:
+            raise koji.GenericError("Cannot free task %i, state is %s" %
+                                    (self.id, koji.TASK_STATES[oldstate]))
+        newstate = koji.TASK_STATES['FREE']
+        newhost = None
+        update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
+                                 data={'state': newstate, 'host_id': newhost})
+        update.execute()
+        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES['FREE'])
+        self.runCallbacks('postTaskStateChange', info, 'host_id', None)
+        return True
+
+    def setWeight(self, weight):
+        """Set weight for task"""
+        weight = convert_value(weight, cast=float)
+        info = self.getInfo(request=True)
+        self.runCallbacks('preTaskStateChange', info, 'weight', weight)
+        # access checks should be performed by calling function
+        update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
+                                 data={'weight': weight})
+        update.execute()
+        self.runCallbacks('postTaskStateChange', info, 'weight', weight)
+
+    def setPriority(self, priority, recurse=False):
+        """Set priority for task"""
+        priority = convert_value(priority, cast=int)
+        info = self.getInfo(request=True)
+        self.runCallbacks('preTaskStateChange', info, 'priority', priority)
+        # access checks should be performed by calling function
+        update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
+                                 data={'priority': priority})
+        update.execute()
+        self.runCallbacks('postTaskStateChange', info, 'priority', priority)
+
+        if recurse:
+            # Change priority of child tasks
+            query = QueryProcessor(tables=['task'], columns=['id'],
+                                   clauses=['parent = %(task_id)s'],
+                                   values={'task_id': self.id},
+                                   opts={'asList': True})
+            for (child_id,) in query.execute():
+                Task(child_id).setPriority(priority, recurse=True)
+
+    def _close(self, result, state):
+        """Mark task closed and set response
+
+        Returns True if successful, False if not"""
+        # access checks should be performed by calling function
+        # this is an approximation, and will be different than what is in the database
+        # the actual value should be retrieved from the 'new' value of the post callback
+        now = time.time()
+        info = self.getInfo(request=True)
+        info['result'] = result
+        self.runCallbacks('preTaskStateChange', info, 'state', state)
+        self.runCallbacks('preTaskStateChange', info, 'completion_ts', now)
+        # get the result from the info dict, so callbacks have a chance to modify it
+        update = UpdateProcessor('task', clauses=['id = %(task_id)d'],
+                                 values={'task_id': self.id},
+                                 data={'result': info['result'], 'state': state},
+                                 rawdata={'completion_time': 'NOW()'})
+        update.execute()
+
+        self.runCallbacks('postTaskStateChange', info, 'state', state)
+        self.runCallbacks('postTaskStateChange', info, 'completion_ts', now)
+
+    def close(self, result):
+        # access checks should be performed by calling function
+        self._close(result, koji.TASK_STATES['CLOSED'])
+
+    def fail(self, result):
+        # access checks should be performed by calling function
+        self._close(result, koji.TASK_STATES['FAILED'])
+
+    def getState(self):
+        query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(id)i'],
+                               values=vars(self))
+        return query.singleValue()
+
+    def isFinished(self):
+        return (koji.TASK_STATES[self.getState()] in ['CLOSED', 'CANCELED', 'FAILED'])
+
+    def isCanceled(self):
+        return (self.getState() == koji.TASK_STATES['CANCELED'])
+
+    def isFailed(self):
+        return (self.getState() == koji.TASK_STATES['FAILED'])
+
+    def cancel(self, recurse=True):
+        """Cancel this task.
+
+        A task can only be canceled if it is not already in the 'CLOSED' state.
+        If it is, no action will be taken.  Return True if the task is
+        successfully canceled, or if it was already canceled, False if it is
+        closed."""
+        # access checks should be performed by calling function
+        now = time.time()
+        info = self.getInfo(request=True)
+        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES['CANCELED'])
+        self.runCallbacks('preTaskStateChange', info, 'completion_ts', now)
+        query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(task_id)s'],
+                               values={'task_id': self.id}, opts={'rowlock': True})
+        state = query.singleValue()
+        st_canceled = koji.TASK_STATES['CANCELED']
+        st_closed = koji.TASK_STATES['CLOSED']
+        st_failed = koji.TASK_STATES['FAILED']
+        if state == st_canceled:
+            return True
+        elif state in [st_closed, st_failed]:
+            return False
+        update = UpdateProcessor('task', clauses=['id = %(task_id)i'], values={'task_id': self.id},
+                                 data={'state': st_canceled}, rawdata={'completion_time': 'NOW()'})
+        update.execute()
+        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES['CANCELED'])
+        self.runCallbacks('postTaskStateChange', info, 'completion_ts', now)
+        # cancel associated builds (only if state is 'BUILDING')
+        # since we check build state, we avoid loops with cancel_build on our end
+        b_building = koji.BUILD_STATES['BUILDING']
+        query = QueryProcessor(tables=['build'], columns=['id'],
+                               clauses=['task_id = %(task_id)i', 'state = %(b_building)i'],
+                               values={'task_id': self.id, 'b_building': b_building},
+                               opts={'rowlock': True, 'asList': True})
+        for (build_id,) in query.execute():
+            cancel_build(build_id, cancel_task=False)
+        if recurse:
+            # also cancel child tasks
+            self.cancelChildren()
+        return True
+
+    def cancelChildren(self):
+        """Cancel child tasks"""
+        query = QueryProcessor(tables=['task'], columns=['id'], clauses=['parent = %(task_id)i'],
+                               values={'task_id': self.id}, opts={'asList': True})
+        for (id,) in query.execute():
+            Task(id).cancel(recurse=True)
+
+    def cancelFull(self, strict=True):
+        """Cancel this task and every other task in its group
+
+        If strict is true, then this must be a top-level task
+        Otherwise we will follow up the chain to find the top-level task
+        """
+        task_id = self.id
+        query = QueryProcessor(tables=['task'], columns=['parent'],
+                               clauses=['id = %(task_id)i'],
+                               values={'task_id': task_id}, opts={'rowlock': True})
+        parent = query.singleValue(strict=False)
+        if parent is not None:
+            if strict:
+                raise koji.GenericError("Task %d is not top-level (parent=%d)" % (task_id, parent))
+            # otherwise, find the top-level task and go from there
+            seen = {task_id: 1}
+            while parent is not None:
+                if parent in seen:
+                    raise koji.GenericError("Task LOOP at task %i" % task_id)
+                task_id = parent
+                seen[task_id] = 1
+                query.values = {'task_id': task_id}
+                parent = query.singleValue()
+            return Task(task_id).cancelFull(strict=True)
+        # We handle the recursion ourselves, since self.cancel will stop at
+        # canceled or closed tasks.
+        tasklist = [task_id]
+        seen = {}
+        # query for use in loop
+        for task_id in tasklist:
+            if task_id in seen:
+                # shouldn't happen
+                raise koji.GenericError("Task LOOP at task %i" % task_id)
+            seen[task_id] = 1
+            Task(task_id).cancel(recurse=False)
+            query = QueryProcessor(tables=['task'], columns=['id'],
+                                   clauses=['parent = %(task_id)i'],
+                                   values={'task_id': task_id}, opts={'asList': True})
+            for (child_id,) in query.execute():
+                tasklist.append(child_id)
+
+    def getRequest(self):
+        query = QueryProcessor(columns=['request'], tables=['task'],
+                               clauses=['id = %(id)i'], values={'id': self.id})
+        xml_request = query.singleValue()
+        if xml_request.find('<?xml', 0, 10) == -1:
+            # handle older base64 encoded data
+            xml_request = base64.b64decode(xml_request)
+        # note: loads accepts either bytes or string
+        params, method = xmlrpc.client.loads(xml_request)
+        return params
+
+    def getResult(self, raise_fault=True):
+        query = QueryProcessor(tables=['task'], columns=['state', 'result'],
+                               clauses=['id = %(id)i'], values={'id': self.id})
+        r = query.executeOne()
+        if not r:
+            raise koji.GenericError("No such task")
+        state = r['state']
+        xml_result = r['result']
+        if koji.TASK_STATES[state] == 'CANCELED':
+            raise koji.GenericError("Task %i is canceled" % self.id)
+        elif koji.TASK_STATES[state] not in ['CLOSED', 'FAILED']:
+            raise koji.GenericError("Task %i is not finished" % self.id)
+        if xml_result.find('<?xml', 0, 10) == -1:
+            # handle older base64 encoded data
+            xml_result = base64.b64decode(xml_result)
+        try:
+            # If the result is a Fault, then loads will raise it
+            # This is normally what we want to happen
+            result, method = xmlrpc.client.loads(xml_result)
+        except xmlrpc.client.Fault as fault:
+            if raise_fault:
+                raise
+            # Note that you can't really return a fault over xmlrpc, except by
+            # raising it. We return a dictionary in the same format that
+            # multiCall does.
+            return {'faultCode': fault.faultCode, 'faultString': fault.faultString}
+        return result[0]
+
+    def getInfo(self, strict=True, request=False):
+        """Return information about the task in a dictionary.  If "request" is True,
+        the request will be decoded and included in the dictionary."""
+        columns, aliases = self._split_fields()
+        query = QueryProcessor(columns=columns, aliases=aliases,
+                               tables=['task'], clauses=['id = %(id)i'],
+                               values={'id': self.id})
+        result = query.executeOne(strict=strict)
+        if result and request:
+            result['request'] = self.getRequest()
+        return result
+
+    def getChildren(self, request=False):
+        """Return information about tasks with this task as their
+        parent.  If there are no such Tasks, return an empty list."""
+        fields = self.fields
+        if request:
+            fields = fields + (('request', 'request'),)
+        columns, aliases = self._split_fields(fields)
+        query = QueryProcessor(columns=columns, aliases=aliases,
+                               tables=['task'], clauses=['parent = %(id)i'],
+                               values={'id': self.id})
+        results = query.execute()
+        if request:
+            for task in results:
+                if task['request'].find('<?xml', 0, 10) == -1:
+                    # handle older base64 encoded data
+                    task['request'] = base64.b64decode(task['request'])
+                # note: loads accepts either bytes or string
+                task['request'] = xmlrpc.client.loads(task['request'])[0]
+        return results
+
+    def runCallbacks(self, cbtype, old_info, attr, new_val):
+        if cbtype.startswith('pre'):
+            info = old_info
+        elif cbtype.startswith('post'):
+            info = self.getInfo(request=True)
+            if info['state'] == koji.TASK_STATES['CLOSED']:
+                # if task is closed, include the result as well
+                info['result'] = self.getResult()
+            new_val = info[attr]
+        else:
+            raise koji.GenericError('No such callback type: %s' % cbtype)
+        old_val = old_info[attr]
+        if attr == 'state':
+            # state is passed in as an integer, but we want to use the string
+            old_val = koji.TASK_STATES[old_val]
+            new_val = koji.TASK_STATES[new_val]
+        koji.plugin.run_callbacks(cbtype, attribute=attr, old=old_val, new=new_val,
+                                  info=info)

From 5fa99482f6b91ef37e47db2bca61ceeb5acef0e4 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 38/92] ...


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 7322834..e0c3584 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -93,6 +93,7 @@ from .db import (  # noqa: F401
     nextval,
     currval,
 )
+from .task import Task
 from .util import convert_value
 
 
diff --git a/kojihub/task.py b/kojihub/task.py
index e515b38..db0dd39 100644
--- a/kojihub/task.py
+++ b/kojihub/task.py
@@ -1,4 +1,17 @@
 # Task related hub code
+import base64
+import logging
+import time
+import xmlrpc.client
+
+import koji
+from .db import QueryProcessor, UpdateProcessor
+from .util import convert_value
+from koji.context import context
+from koji.util import decode_bytes
+
+
+logger = logging.getLogger('koji.hub.task')
 
 
 class Task(object):
@@ -106,12 +119,13 @@ class Task(object):
             otherhost = r['host_id']
             if state == koji.TASK_STATES['FREE']:
                 if otherhost is not None:
-                    log_error(f"Error: task {task_id} is both free "
-                              f"and handled by host {otherhost}")
+                    logger.error(f"Error: task {task_id} is both free "
+                                 f"and handled by host {otherhost}")
                     return False
             elif state == koji.TASK_STATES['ASSIGNED']:
                 if otherhost is None:
-                    log_error(f"Error: task {task_id} is assigned, but no host is really assigned")
+                    logger.error(f"Error: task {task_id} is assigned, but no host is really "
+                                 "assigned")
                     return False
                 elif otherhost != host_id:
                     # task is assigned to someone else, no error just return
@@ -119,7 +133,7 @@ class Task(object):
                 elif newstate == 'ASSIGNED':
                     # double assign is a weird situation but we can return True as state doesn't
                     # really change
-                    log_error(f"Error: double assign of task {task_id} and host {host_id}")
+                    logger.error(f"Error: double assign of task {task_id} and host {host_id}")
                     return True
                 # otherwise the task is assigned to host_id, so keep going
             elif state == koji.TASK_STATES['CANCELED']:
@@ -127,16 +141,16 @@ class Task(object):
                 return False
             elif state == koji.TASK_STATES['OPEN']:
                 if otherhost is None:
-                    log_error(f"Error: task {task_id} is opened but not handled by any host")
+                    logger.error(f"Error: task {task_id} is opened but not handled by any host")
                 elif otherhost == host_id:
-                    log_error(f"Error: task {task_id} is already open and handled by "
-                              f"{host_id} (double open/assign)")
+                    logger.error(f"Error: task {task_id} is already open and handled by "
+                                 f"{host_id} (double open/assign)")
                 return False
             else:
                 # state is CLOSED or FAILED
                 if otherhost is None:
-                    log_error(f"Error: task {task_id} is non-free but not handled by any host "
-                              f"(state {koji.TASK_STATES[state]})")
+                    logger.error(f"Error: task {task_id} is non-free but not handled by any host "
+                                 f"(state {koji.TASK_STATES[state]})")
                 return False
         # if we reach here, task is either
         #  - free and unlocked

From ec21f70e0b31151ebb86c1e99d8cf5c3efd317de Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 39/92] Revert "..."


This reverts commit 8fc33e56070c09b94b5bd689e3871e537c2a8227.

---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index e0c3584..7322834 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -93,7 +93,6 @@ from .db import (  # noqa: F401
     nextval,
     currval,
 )
-from .task import Task
 from .util import convert_value
 
 
diff --git a/kojihub/task.py b/kojihub/task.py
index db0dd39..e515b38 100644
--- a/kojihub/task.py
+++ b/kojihub/task.py
@@ -1,17 +1,4 @@
 # Task related hub code
-import base64
-import logging
-import time
-import xmlrpc.client
-
-import koji
-from .db import QueryProcessor, UpdateProcessor
-from .util import convert_value
-from koji.context import context
-from koji.util import decode_bytes
-
-
-logger = logging.getLogger('koji.hub.task')
 
 
 class Task(object):
@@ -119,13 +106,12 @@ class Task(object):
             otherhost = r['host_id']
             if state == koji.TASK_STATES['FREE']:
                 if otherhost is not None:
-                    logger.error(f"Error: task {task_id} is both free "
-                                 f"and handled by host {otherhost}")
+                    log_error(f"Error: task {task_id} is both free "
+                              f"and handled by host {otherhost}")
                     return False
             elif state == koji.TASK_STATES['ASSIGNED']:
                 if otherhost is None:
-                    logger.error(f"Error: task {task_id} is assigned, but no host is really "
-                                 "assigned")
+                    log_error(f"Error: task {task_id} is assigned, but no host is really assigned")
                     return False
                 elif otherhost != host_id:
                     # task is assigned to someone else, no error just return
@@ -133,7 +119,7 @@ class Task(object):
                 elif newstate == 'ASSIGNED':
                     # double assign is a weird situation but we can return True as state doesn't
                     # really change
-                    logger.error(f"Error: double assign of task {task_id} and host {host_id}")
+                    log_error(f"Error: double assign of task {task_id} and host {host_id}")
                     return True
                 # otherwise the task is assigned to host_id, so keep going
             elif state == koji.TASK_STATES['CANCELED']:
@@ -141,16 +127,16 @@ class Task(object):
                 return False
             elif state == koji.TASK_STATES['OPEN']:
                 if otherhost is None:
-                    logger.error(f"Error: task {task_id} is opened but not handled by any host")
+                    log_error(f"Error: task {task_id} is opened but not handled by any host")
                 elif otherhost == host_id:
-                    logger.error(f"Error: task {task_id} is already open and handled by "
-                                 f"{host_id} (double open/assign)")
+                    log_error(f"Error: task {task_id} is already open and handled by "
+                              f"{host_id} (double open/assign)")
                 return False
             else:
                 # state is CLOSED or FAILED
                 if otherhost is None:
-                    logger.error(f"Error: task {task_id} is non-free but not handled by any host "
-                                 f"(state {koji.TASK_STATES[state]})")
+                    log_error(f"Error: task {task_id} is non-free but not handled by any host "
+                              f"(state {koji.TASK_STATES[state]})")
                 return False
         # if we reach here, task is either
         #  - free and unlocked

From 950a84a0adabab9097a4cf1e399bf5da5af45eec Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 40/92] Revert "move Task class"


This reverts commit e385df81fb90b4cb15928ff64fac239ce90ca4fd.

---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 7322834..016bb11 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -112,6 +112,460 @@ def xform_user_krb(entry):
     return entry
 
 
+class Task(object):
+    """A task for the build hosts"""
+
+    fields = (
+        ('task.id', 'id'),
+        ('task.state', 'state'),
+        ('task.create_time', 'create_time'),
+        ("date_part('epoch', create_time)", 'create_ts'),
+        ('task.start_time', 'start_time'),
+        ("date_part('epoch', task.start_time)", 'start_ts'),
+        ('task.completion_time', 'completion_time'),
+        ("date_part('epoch', completion_time)", 'completion_ts'),
+        ('task.channel_id', 'channel_id'),
+        ('task.host_id', 'host_id'),
+        ('task.parent', 'parent'),
+        ('task.label', 'label'),
+        ('task.waiting', 'waiting'),
+        ('task.awaited', 'awaited'),
+        ('task.owner', 'owner'),
+        ('task.method', 'method'),
+        ('task.arch', 'arch'),
+        ('task.priority', 'priority'),
+        ('task.weight', 'weight'))
+
+    def __init__(self, id):
+        self.id = convert_value(id, cast=int)
+        self.logger = logging.getLogger("koji.hub.Task")
+
+    def _split_fields(self, fields=None):
+        """Helper function for split fields to QueryProcessor's
+       columns/aliases options"""
+        if fields is None:
+            fields = self.fields
+        columns = [f[0] for f in fields]
+        aliases = [f[1] for f in fields]
+        return columns, aliases
+
+    def verifyHost(self, host_id=None):
+        """Verify that host owns task"""
+        if host_id is None:
+            host_id = context.session.host_id
+        if host_id is None:
+            return False
+        task_id = self.id
+        # getting a row lock on this task to ensure task assignment sanity
+        # no other concurrent transaction should be altering this row
+        query = QueryProcessor(tables=['task'], columns=['state', 'host_id'],
+                               clauses=['id=%(task_id)s'], values={'task_id': task_id},
+                               opts={'rowlock': True})
+        r = query.executeOne()
+        if not r:
+            raise koji.GenericError("No such task: %i" % task_id)
+        return (r['state'] == koji.TASK_STATES['OPEN'] and r['host_id'] == host_id)
+
+    def assertHost(self, host_id):
+        if not self.verifyHost(host_id):
+            raise koji.ActionNotAllowed("host %d does not own task %d" % (host_id, self.id))
+
+    def getOwner(self):
+        """Return the owner (user_id) for this task"""
+        query = QueryProcessor(tables=['task'], columns=['owner'],
+                               clauses=['id=%(id)i'], values=vars(self))
+        return query.singleValue()
+
+    def verifyOwner(self, user_id=None):
+        """Verify that user owns task"""
+        if user_id is None:
+            user_id = context.session.user_id
+        if user_id is None:
+            return False
+        task_id = self.id
+        # getting a row lock on this task to ensure task state sanity
+        query = QueryProcessor(tables=['task'], columns=['owner'],
+                               clauses=['id=%(task_id)s'], values={'task_id': task_id},
+                               opts={'rowlock': True})
+        owner = query.singleValue(strict=False)
+        if not owner:
+            raise koji.GenericError("No such task: %i" % task_id)
+        return (owner == user_id)
+
+    def assertOwner(self, user_id=None):
+        if not self.verifyOwner(user_id):
+            raise koji.ActionNotAllowed("user %d does not own task %d" % (user_id, self.id))
+
+    def lock(self, host_id, newstate='OPEN', force=False):
+        """Attempt to associate the task for host, either to assign or open
+
+        returns True if successful, False otherwise"""
+        info = self.getInfo(request=True)
+        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES[newstate])
+        self.runCallbacks('preTaskStateChange', info, 'host_id', host_id)
+        # we use row-level locks to keep things sane
+        # note the QueryProcessor...opts={'rowlock': True}
+        task_id = self.id
+        if not force:
+            query = QueryProcessor(columns=['state', 'host_id'], tables=['task'],
+                                   clauses=['id=%(task_id)s'], values={'task_id': task_id},
+                                   opts={'rowlock': True})
+            r = query.executeOne()
+            if not r:
+                raise koji.GenericError("No such task: %i" % task_id)
+            state = r['state']
+            otherhost = r['host_id']
+            if state == koji.TASK_STATES['FREE']:
+                if otherhost is not None:
+                    log_error(f"Error: task {task_id} is both free "
+                              f"and handled by host {otherhost}")
+                    return False
+            elif state == koji.TASK_STATES['ASSIGNED']:
+                if otherhost is None:
+                    log_error(f"Error: task {task_id} is assigned, but no host is really assigned")
+                    return False
+                elif otherhost != host_id:
+                    # task is assigned to someone else, no error just return
+                    return False
+                elif newstate == 'ASSIGNED':
+                    # double assign is a weird situation but we can return True as state doesn't
+                    # really change
+                    log_error(f"Error: double assign of task {task_id} and host {host_id}")
+                    return True
+                # otherwise the task is assigned to host_id, so keep going
+            elif state == koji.TASK_STATES['CANCELED']:
+                # it is ok that task was canceled meanwhile
+                return False
+            elif state == koji.TASK_STATES['OPEN']:
+                if otherhost is None:
+                    log_error(f"Error: task {task_id} is opened but not handled by any host")
+                elif otherhost == host_id:
+                    log_error(f"Error: task {task_id} is already open and handled by "
+                              f"{host_id} (double open/assign)")
+                return False
+            else:
+                # state is CLOSED or FAILED
+                if otherhost is None:
+                    log_error(f"Error: task {task_id} is non-free but not handled by any host "
+                              f"(state {koji.TASK_STATES[state]})")
+                return False
+        # if we reach here, task is either
+        #  - free and unlocked
+        #  - assigned to host_id
+        #  - force option is enabled
+        state = koji.TASK_STATES[newstate]
+        update = UpdateProcessor('task', clauses=['id=%(task_id)i'], values=locals())
+        update.set(state=state, host_id=host_id)
+        if state == koji.TASK_STATES['OPEN']:
+            update.rawset(start_time='NOW()')
+        update.execute()
+        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES[newstate])
+        self.runCallbacks('postTaskStateChange', info, 'host_id', host_id)
+        return True
+
+    def assign(self, host_id, force=False):
+        """Attempt to assign the task to host.
+
+        returns True if successful, False otherwise"""
+        return self.lock(host_id, 'ASSIGNED', force)
+
+    def open(self, host_id):
+        """Attempt to open the task for host.
+
+        returns task data if successful, None otherwise"""
+        if self.lock(host_id, 'OPEN'):
+            # get more complete data to return
+            fields = self.fields + (('task.request', 'request'),)
+            query = QueryProcessor(tables=['task'], clauses=['id=%(id)i'], values=vars(self),
+                                   columns=[f[0] for f in fields], aliases=[f[1] for f in fields])
+            ret = query.executeOne()
+            if ret['request'].find('<?xml', 0, 10) == -1:
+                # handle older base64 encoded data
+                data = base64.b64decode(ret['request'])
+                # we can't return raw bytes and this /should/ be a valid string
+                ret['request'] = decode_bytes(data)
+            return ret
+        else:
+            return None
+
+    def free(self):
+        """Free a task"""
+        info = self.getInfo(request=True)
+        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES['FREE'])
+        self.runCallbacks('preTaskStateChange', info, 'host_id', None)
+        # access checks should be performed by calling function
+        query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(id)i'],
+                               values=vars(self), opts={'rowlock': True})
+        oldstate = query.singleValue(strict=False)
+        if not oldstate:
+            raise koji.GenericError("No such task: %i" % self.id)
+        if koji.TASK_STATES[oldstate] in ['CLOSED', 'CANCELED', 'FAILED']:
+            raise koji.GenericError("Cannot free task %i, state is %s" %
+                                    (self.id, koji.TASK_STATES[oldstate]))
+        newstate = koji.TASK_STATES['FREE']
+        newhost = None
+        update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
+                                 data={'state': newstate, 'host_id': newhost})
+        update.execute()
+        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES['FREE'])
+        self.runCallbacks('postTaskStateChange', info, 'host_id', None)
+        return True
+
+    def setWeight(self, weight):
+        """Set weight for task"""
+        weight = convert_value(weight, cast=float)
+        info = self.getInfo(request=True)
+        self.runCallbacks('preTaskStateChange', info, 'weight', weight)
+        # access checks should be performed by calling function
+        update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
+                                 data={'weight': weight})
+        update.execute()
+        self.runCallbacks('postTaskStateChange', info, 'weight', weight)
+
+    def setPriority(self, priority, recurse=False):
+        """Set priority for task"""
+        priority = convert_value(priority, cast=int)
+        info = self.getInfo(request=True)
+        self.runCallbacks('preTaskStateChange', info, 'priority', priority)
+        # access checks should be performed by calling function
+        update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
+                                 data={'priority': priority})
+        update.execute()
+        self.runCallbacks('postTaskStateChange', info, 'priority', priority)
+
+        if recurse:
+            # Change priority of child tasks
+            query = QueryProcessor(tables=['task'], columns=['id'],
+                                   clauses=['parent = %(task_id)s'],
+                                   values={'task_id': self.id},
+                                   opts={'asList': True})
+            for (child_id,) in query.execute():
+                Task(child_id).setPriority(priority, recurse=True)
+
+    def _close(self, result, state):
+        """Mark task closed and set response
+
+        Returns True if successful, False if not"""
+        # access checks should be performed by calling function
+        # this is an approximation, and will be different than what is in the database
+        # the actual value should be retrieved from the 'new' value of the post callback
+        now = time.time()
+        info = self.getInfo(request=True)
+        info['result'] = result
+        self.runCallbacks('preTaskStateChange', info, 'state', state)
+        self.runCallbacks('preTaskStateChange', info, 'completion_ts', now)
+        # get the result from the info dict, so callbacks have a chance to modify it
+        update = UpdateProcessor('task', clauses=['id = %(task_id)d'],
+                                 values={'task_id': self.id},
+                                 data={'result': info['result'], 'state': state},
+                                 rawdata={'completion_time': 'NOW()'})
+        update.execute()
+
+        self.runCallbacks('postTaskStateChange', info, 'state', state)
+        self.runCallbacks('postTaskStateChange', info, 'completion_ts', now)
+
+    def close(self, result):
+        # access checks should be performed by calling function
+        self._close(result, koji.TASK_STATES['CLOSED'])
+
+    def fail(self, result):
+        # access checks should be performed by calling function
+        self._close(result, koji.TASK_STATES['FAILED'])
+
+    def getState(self):
+        query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(id)i'],
+                               values=vars(self))
+        return query.singleValue()
+
+    def isFinished(self):
+        return (koji.TASK_STATES[self.getState()] in ['CLOSED', 'CANCELED', 'FAILED'])
+
+    def isCanceled(self):
+        return (self.getState() == koji.TASK_STATES['CANCELED'])
+
+    def isFailed(self):
+        return (self.getState() == koji.TASK_STATES['FAILED'])
+
+    def cancel(self, recurse=True):
+        """Cancel this task.
+
+        A task can only be canceled if it is not already in the 'CLOSED' state.
+        If it is, no action will be taken.  Return True if the task is
+        successfully canceled, or if it was already canceled, False if it is
+        closed."""
+        # access checks should be performed by calling function
+        now = time.time()
+        info = self.getInfo(request=True)
+        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES['CANCELED'])
+        self.runCallbacks('preTaskStateChange', info, 'completion_ts', now)
+        query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(task_id)s'],
+                               values={'task_id': self.id}, opts={'rowlock': True})
+        state = query.singleValue()
+        st_canceled = koji.TASK_STATES['CANCELED']
+        st_closed = koji.TASK_STATES['CLOSED']
+        st_failed = koji.TASK_STATES['FAILED']
+        if state == st_canceled:
+            return True
+        elif state in [st_closed, st_failed]:
+            return False
+        update = UpdateProcessor('task', clauses=['id = %(task_id)i'], values={'task_id': self.id},
+                                 data={'state': st_canceled}, rawdata={'completion_time': 'NOW()'})
+        update.execute()
+        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES['CANCELED'])
+        self.runCallbacks('postTaskStateChange', info, 'completion_ts', now)
+        # cancel associated builds (only if state is 'BUILDING')
+        # since we check build state, we avoid loops with cancel_build on our end
+        b_building = koji.BUILD_STATES['BUILDING']
+        query = QueryProcessor(tables=['build'], columns=['id'],
+                               clauses=['task_id = %(task_id)i', 'state = %(b_building)i'],
+                               values={'task_id': self.id, 'b_building': b_building},
+                               opts={'rowlock': True, 'asList': True})
+        for (build_id,) in query.execute():
+            cancel_build(build_id, cancel_task=False)
+        if recurse:
+            # also cancel child tasks
+            self.cancelChildren()
+        return True
+
+    def cancelChildren(self):
+        """Cancel child tasks"""
+        query = QueryProcessor(tables=['task'], columns=['id'], clauses=['parent = %(task_id)i'],
+                               values={'task_id': self.id}, opts={'asList': True})
+        for (id,) in query.execute():
+            Task(id).cancel(recurse=True)
+
+    def cancelFull(self, strict=True):
+        """Cancel this task and every other task in its group
+
+        If strict is true, then this must be a top-level task
+        Otherwise we will follow up the chain to find the top-level task
+        """
+        task_id = self.id
+        query = QueryProcessor(tables=['task'], columns=['parent'],
+                               clauses=['id = %(task_id)i'],
+                               values={'task_id': task_id}, opts={'rowlock': True})
+        parent = query.singleValue(strict=False)
+        if parent is not None:
+            if strict:
+                raise koji.GenericError("Task %d is not top-level (parent=%d)" % (task_id, parent))
+            # otherwise, find the top-level task and go from there
+            seen = {task_id: 1}
+            while parent is not None:
+                if parent in seen:
+                    raise koji.GenericError("Task LOOP at task %i" % task_id)
+                task_id = parent
+                seen[task_id] = 1
+                query.values = {'task_id': task_id}
+                parent = query.singleValue()
+            return Task(task_id).cancelFull(strict=True)
+        # We handle the recursion ourselves, since self.cancel will stop at
+        # canceled or closed tasks.
+        tasklist = [task_id]
+        seen = {}
+        # query for use in loop
+        for task_id in tasklist:
+            if task_id in seen:
+                # shouldn't happen
+                raise koji.GenericError("Task LOOP at task %i" % task_id)
+            seen[task_id] = 1
+            Task(task_id).cancel(recurse=False)
+            query = QueryProcessor(tables=['task'], columns=['id'],
+                                   clauses=['parent = %(task_id)i'],
+                                   values={'task_id': task_id}, opts={'asList': True})
+            for (child_id,) in query.execute():
+                tasklist.append(child_id)
+
+    def getRequest(self):
+        query = QueryProcessor(columns=['request'], tables=['task'],
+                               clauses=['id = %(id)i'], values={'id': self.id})
+        xml_request = query.singleValue()
+        if xml_request.find('<?xml', 0, 10) == -1:
+            # handle older base64 encoded data
+            xml_request = base64.b64decode(xml_request)
+        # note: loads accepts either bytes or string
+        params, method = xmlrpc.client.loads(xml_request)
+        return params
+
+    def getResult(self, raise_fault=True):
+        query = QueryProcessor(tables=['task'], columns=['state', 'result'],
+                               clauses=['id = %(id)i'], values={'id': self.id})
+        r = query.executeOne()
+        if not r:
+            raise koji.GenericError("No such task")
+        state = r['state']
+        xml_result = r['result']
+        if koji.TASK_STATES[state] == 'CANCELED':
+            raise koji.GenericError("Task %i is canceled" % self.id)
+        elif koji.TASK_STATES[state] not in ['CLOSED', 'FAILED']:
+            raise koji.GenericError("Task %i is not finished" % self.id)
+        if xml_result.find('<?xml', 0, 10) == -1:
+            # handle older base64 encoded data
+            xml_result = base64.b64decode(xml_result)
+        try:
+            # If the result is a Fault, then loads will raise it
+            # This is normally what we want to happen
+            result, method = xmlrpc.client.loads(xml_result)
+        except xmlrpc.client.Fault as fault:
+            if raise_fault:
+                raise
+            # Note that you can't really return a fault over xmlrpc, except by
+            # raising it. We return a dictionary in the same format that
+            # multiCall does.
+            return {'faultCode': fault.faultCode, 'faultString': fault.faultString}
+        return result[0]
+
+    def getInfo(self, strict=True, request=False):
+        """Return information about the task in a dictionary.  If "request" is True,
+        the request will be decoded and included in the dictionary."""
+        columns, aliases = self._split_fields()
+        query = QueryProcessor(columns=columns, aliases=aliases,
+                               tables=['task'], clauses=['id = %(id)i'],
+                               values={'id': self.id})
+        result = query.executeOne(strict=strict)
+        if result and request:
+            result['request'] = self.getRequest()
+        return result
+
+    def getChildren(self, request=False):
+        """Return information about tasks with this task as their
+        parent.  If there are no such Tasks, return an empty list."""
+        fields = self.fields
+        if request:
+            fields = fields + (('request', 'request'),)
+        columns, aliases = self._split_fields(fields)
+        query = QueryProcessor(columns=columns, aliases=aliases,
+                               tables=['task'], clauses=['parent = %(id)i'],
+                               values={'id': self.id})
+        results = query.execute()
+        if request:
+            for task in results:
+                if task['request'].find('<?xml', 0, 10) == -1:
+                    # handle older base64 encoded data
+                    task['request'] = base64.b64decode(task['request'])
+                # note: loads accepts either bytes or string
+                task['request'] = xmlrpc.client.loads(task['request'])[0]
+        return results
+
+    def runCallbacks(self, cbtype, old_info, attr, new_val):
+        if cbtype.startswith('pre'):
+            info = old_info
+        elif cbtype.startswith('post'):
+            info = self.getInfo(request=True)
+            if info['state'] == koji.TASK_STATES['CLOSED']:
+                # if task is closed, include the result as well
+                info['result'] = self.getResult()
+            new_val = info[attr]
+        else:
+            raise koji.GenericError('No such callback type: %s' % cbtype)
+        old_val = old_info[attr]
+        if attr == 'state':
+            # state is passed in as an integer, but we want to use the string
+            old_val = koji.TASK_STATES[old_val]
+            new_val = koji.TASK_STATES[new_val]
+        koji.plugin.run_callbacks(cbtype, attribute=attr, old=old_val, new=new_val,
+                                  info=info)
+
+
 def make_task(method, arglist, **opts):
     """Create a task
 
diff --git a/kojihub/task.py b/kojihub/task.py
deleted file mode 100644
index e515b38..0000000
--- a/kojihub/task.py
+++ /dev/null
@@ -1,455 +0,0 @@
-# Task related hub code
-
-
-class Task(object):
-    """A task for the build hosts"""
-
-    fields = (
-        ('task.id', 'id'),
-        ('task.state', 'state'),
-        ('task.create_time', 'create_time'),
-        ("date_part('epoch', create_time)", 'create_ts'),
-        ('task.start_time', 'start_time'),
-        ("date_part('epoch', task.start_time)", 'start_ts'),
-        ('task.completion_time', 'completion_time'),
-        ("date_part('epoch', completion_time)", 'completion_ts'),
-        ('task.channel_id', 'channel_id'),
-        ('task.host_id', 'host_id'),
-        ('task.parent', 'parent'),
-        ('task.label', 'label'),
-        ('task.waiting', 'waiting'),
-        ('task.awaited', 'awaited'),
-        ('task.owner', 'owner'),
-        ('task.method', 'method'),
-        ('task.arch', 'arch'),
-        ('task.priority', 'priority'),
-        ('task.weight', 'weight'))
-
-    def __init__(self, id):
-        self.id = convert_value(id, cast=int)
-        self.logger = logging.getLogger("koji.hub.Task")
-
-    def _split_fields(self, fields=None):
-        """Helper function for split fields to QueryProcessor's
-       columns/aliases options"""
-        if fields is None:
-            fields = self.fields
-        columns = [f[0] for f in fields]
-        aliases = [f[1] for f in fields]
-        return columns, aliases
-
-    def verifyHost(self, host_id=None):
-        """Verify that host owns task"""
-        if host_id is None:
-            host_id = context.session.host_id
-        if host_id is None:
-            return False
-        task_id = self.id
-        # getting a row lock on this task to ensure task assignment sanity
-        # no other concurrent transaction should be altering this row
-        query = QueryProcessor(tables=['task'], columns=['state', 'host_id'],
-                               clauses=['id=%(task_id)s'], values={'task_id': task_id},
-                               opts={'rowlock': True})
-        r = query.executeOne()
-        if not r:
-            raise koji.GenericError("No such task: %i" % task_id)
-        return (r['state'] == koji.TASK_STATES['OPEN'] and r['host_id'] == host_id)
-
-    def assertHost(self, host_id):
-        if not self.verifyHost(host_id):
-            raise koji.ActionNotAllowed("host %d does not own task %d" % (host_id, self.id))
-
-    def getOwner(self):
-        """Return the owner (user_id) for this task"""
-        query = QueryProcessor(tables=['task'], columns=['owner'],
-                               clauses=['id=%(id)i'], values=vars(self))
-        return query.singleValue()
-
-    def verifyOwner(self, user_id=None):
-        """Verify that user owns task"""
-        if user_id is None:
-            user_id = context.session.user_id
-        if user_id is None:
-            return False
-        task_id = self.id
-        # getting a row lock on this task to ensure task state sanity
-        query = QueryProcessor(tables=['task'], columns=['owner'],
-                               clauses=['id=%(task_id)s'], values={'task_id': task_id},
-                               opts={'rowlock': True})
-        owner = query.singleValue(strict=False)
-        if not owner:
-            raise koji.GenericError("No such task: %i" % task_id)
-        return (owner == user_id)
-
-    def assertOwner(self, user_id=None):
-        if not self.verifyOwner(user_id):
-            raise koji.ActionNotAllowed("user %d does not own task %d" % (user_id, self.id))
-
-    def lock(self, host_id, newstate='OPEN', force=False):
-        """Attempt to associate the task for host, either to assign or open
-
-        returns True if successful, False otherwise"""
-        info = self.getInfo(request=True)
-        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES[newstate])
-        self.runCallbacks('preTaskStateChange', info, 'host_id', host_id)
-        # we use row-level locks to keep things sane
-        # note the QueryProcessor...opts={'rowlock': True}
-        task_id = self.id
-        if not force:
-            query = QueryProcessor(columns=['state', 'host_id'], tables=['task'],
-                                   clauses=['id=%(task_id)s'], values={'task_id': task_id},
-                                   opts={'rowlock': True})
-            r = query.executeOne()
-            if not r:
-                raise koji.GenericError("No such task: %i" % task_id)
-            state = r['state']
-            otherhost = r['host_id']
-            if state == koji.TASK_STATES['FREE']:
-                if otherhost is not None:
-                    log_error(f"Error: task {task_id} is both free "
-                              f"and handled by host {otherhost}")
-                    return False
-            elif state == koji.TASK_STATES['ASSIGNED']:
-                if otherhost is None:
-                    log_error(f"Error: task {task_id} is assigned, but no host is really assigned")
-                    return False
-                elif otherhost != host_id:
-                    # task is assigned to someone else, no error just return
-                    return False
-                elif newstate == 'ASSIGNED':
-                    # double assign is a weird situation but we can return True as state doesn't
-                    # really change
-                    log_error(f"Error: double assign of task {task_id} and host {host_id}")
-                    return True
-                # otherwise the task is assigned to host_id, so keep going
-            elif state == koji.TASK_STATES['CANCELED']:
-                # it is ok that task was canceled meanwhile
-                return False
-            elif state == koji.TASK_STATES['OPEN']:
-                if otherhost is None:
-                    log_error(f"Error: task {task_id} is opened but not handled by any host")
-                elif otherhost == host_id:
-                    log_error(f"Error: task {task_id} is already open and handled by "
-                              f"{host_id} (double open/assign)")
-                return False
-            else:
-                # state is CLOSED or FAILED
-                if otherhost is None:
-                    log_error(f"Error: task {task_id} is non-free but not handled by any host "
-                              f"(state {koji.TASK_STATES[state]})")
-                return False
-        # if we reach here, task is either
-        #  - free and unlocked
-        #  - assigned to host_id
-        #  - force option is enabled
-        state = koji.TASK_STATES[newstate]
-        update = UpdateProcessor('task', clauses=['id=%(task_id)i'], values=locals())
-        update.set(state=state, host_id=host_id)
-        if state == koji.TASK_STATES['OPEN']:
-            update.rawset(start_time='NOW()')
-        update.execute()
-        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES[newstate])
-        self.runCallbacks('postTaskStateChange', info, 'host_id', host_id)
-        return True
-
-    def assign(self, host_id, force=False):
-        """Attempt to assign the task to host.
-
-        returns True if successful, False otherwise"""
-        return self.lock(host_id, 'ASSIGNED', force)
-
-    def open(self, host_id):
-        """Attempt to open the task for host.
-
-        returns task data if successful, None otherwise"""
-        if self.lock(host_id, 'OPEN'):
-            # get more complete data to return
-            fields = self.fields + (('task.request', 'request'),)
-            query = QueryProcessor(tables=['task'], clauses=['id=%(id)i'], values=vars(self),
-                                   columns=[f[0] for f in fields], aliases=[f[1] for f in fields])
-            ret = query.executeOne()
-            if ret['request'].find('<?xml', 0, 10) == -1:
-                # handle older base64 encoded data
-                data = base64.b64decode(ret['request'])
-                # we can't return raw bytes and this /should/ be a valid string
-                ret['request'] = decode_bytes(data)
-            return ret
-        else:
-            return None
-
-    def free(self):
-        """Free a task"""
-        info = self.getInfo(request=True)
-        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES['FREE'])
-        self.runCallbacks('preTaskStateChange', info, 'host_id', None)
-        # access checks should be performed by calling function
-        query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(id)i'],
-                               values=vars(self), opts={'rowlock': True})
-        oldstate = query.singleValue(strict=False)
-        if not oldstate:
-            raise koji.GenericError("No such task: %i" % self.id)
-        if koji.TASK_STATES[oldstate] in ['CLOSED', 'CANCELED', 'FAILED']:
-            raise koji.GenericError("Cannot free task %i, state is %s" %
-                                    (self.id, koji.TASK_STATES[oldstate]))
-        newstate = koji.TASK_STATES['FREE']
-        newhost = None
-        update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
-                                 data={'state': newstate, 'host_id': newhost})
-        update.execute()
-        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES['FREE'])
-        self.runCallbacks('postTaskStateChange', info, 'host_id', None)
-        return True
-
-    def setWeight(self, weight):
-        """Set weight for task"""
-        weight = convert_value(weight, cast=float)
-        info = self.getInfo(request=True)
-        self.runCallbacks('preTaskStateChange', info, 'weight', weight)
-        # access checks should be performed by calling function
-        update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
-                                 data={'weight': weight})
-        update.execute()
-        self.runCallbacks('postTaskStateChange', info, 'weight', weight)
-
-    def setPriority(self, priority, recurse=False):
-        """Set priority for task"""
-        priority = convert_value(priority, cast=int)
-        info = self.getInfo(request=True)
-        self.runCallbacks('preTaskStateChange', info, 'priority', priority)
-        # access checks should be performed by calling function
-        update = UpdateProcessor('task', clauses=['id=%(task_id)s'], values={'task_id': self.id},
-                                 data={'priority': priority})
-        update.execute()
-        self.runCallbacks('postTaskStateChange', info, 'priority', priority)
-
-        if recurse:
-            # Change priority of child tasks
-            query = QueryProcessor(tables=['task'], columns=['id'],
-                                   clauses=['parent = %(task_id)s'],
-                                   values={'task_id': self.id},
-                                   opts={'asList': True})
-            for (child_id,) in query.execute():
-                Task(child_id).setPriority(priority, recurse=True)
-
-    def _close(self, result, state):
-        """Mark task closed and set response
-
-        Returns True if successful, False if not"""
-        # access checks should be performed by calling function
-        # this is an approximation, and will be different than what is in the database
-        # the actual value should be retrieved from the 'new' value of the post callback
-        now = time.time()
-        info = self.getInfo(request=True)
-        info['result'] = result
-        self.runCallbacks('preTaskStateChange', info, 'state', state)
-        self.runCallbacks('preTaskStateChange', info, 'completion_ts', now)
-        # get the result from the info dict, so callbacks have a chance to modify it
-        update = UpdateProcessor('task', clauses=['id = %(task_id)d'],
-                                 values={'task_id': self.id},
-                                 data={'result': info['result'], 'state': state},
-                                 rawdata={'completion_time': 'NOW()'})
-        update.execute()
-
-        self.runCallbacks('postTaskStateChange', info, 'state', state)
-        self.runCallbacks('postTaskStateChange', info, 'completion_ts', now)
-
-    def close(self, result):
-        # access checks should be performed by calling function
-        self._close(result, koji.TASK_STATES['CLOSED'])
-
-    def fail(self, result):
-        # access checks should be performed by calling function
-        self._close(result, koji.TASK_STATES['FAILED'])
-
-    def getState(self):
-        query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(id)i'],
-                               values=vars(self))
-        return query.singleValue()
-
-    def isFinished(self):
-        return (koji.TASK_STATES[self.getState()] in ['CLOSED', 'CANCELED', 'FAILED'])
-
-    def isCanceled(self):
-        return (self.getState() == koji.TASK_STATES['CANCELED'])
-
-    def isFailed(self):
-        return (self.getState() == koji.TASK_STATES['FAILED'])
-
-    def cancel(self, recurse=True):
-        """Cancel this task.
-
-        A task can only be canceled if it is not already in the 'CLOSED' state.
-        If it is, no action will be taken.  Return True if the task is
-        successfully canceled, or if it was already canceled, False if it is
-        closed."""
-        # access checks should be performed by calling function
-        now = time.time()
-        info = self.getInfo(request=True)
-        self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES['CANCELED'])
-        self.runCallbacks('preTaskStateChange', info, 'completion_ts', now)
-        query = QueryProcessor(tables=['task'], columns=['state'], clauses=['id = %(task_id)s'],
-                               values={'task_id': self.id}, opts={'rowlock': True})
-        state = query.singleValue()
-        st_canceled = koji.TASK_STATES['CANCELED']
-        st_closed = koji.TASK_STATES['CLOSED']
-        st_failed = koji.TASK_STATES['FAILED']
-        if state == st_canceled:
-            return True
-        elif state in [st_closed, st_failed]:
-            return False
-        update = UpdateProcessor('task', clauses=['id = %(task_id)i'], values={'task_id': self.id},
-                                 data={'state': st_canceled}, rawdata={'completion_time': 'NOW()'})
-        update.execute()
-        self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES['CANCELED'])
-        self.runCallbacks('postTaskStateChange', info, 'completion_ts', now)
-        # cancel associated builds (only if state is 'BUILDING')
-        # since we check build state, we avoid loops with cancel_build on our end
-        b_building = koji.BUILD_STATES['BUILDING']
-        query = QueryProcessor(tables=['build'], columns=['id'],
-                               clauses=['task_id = %(task_id)i', 'state = %(b_building)i'],
-                               values={'task_id': self.id, 'b_building': b_building},
-                               opts={'rowlock': True, 'asList': True})
-        for (build_id,) in query.execute():
-            cancel_build(build_id, cancel_task=False)
-        if recurse:
-            # also cancel child tasks
-            self.cancelChildren()
-        return True
-
-    def cancelChildren(self):
-        """Cancel child tasks"""
-        query = QueryProcessor(tables=['task'], columns=['id'], clauses=['parent = %(task_id)i'],
-                               values={'task_id': self.id}, opts={'asList': True})
-        for (id,) in query.execute():
-            Task(id).cancel(recurse=True)
-
-    def cancelFull(self, strict=True):
-        """Cancel this task and every other task in its group
-
-        If strict is true, then this must be a top-level task
-        Otherwise we will follow up the chain to find the top-level task
-        """
-        task_id = self.id
-        query = QueryProcessor(tables=['task'], columns=['parent'],
-                               clauses=['id = %(task_id)i'],
-                               values={'task_id': task_id}, opts={'rowlock': True})
-        parent = query.singleValue(strict=False)
-        if parent is not None:
-            if strict:
-                raise koji.GenericError("Task %d is not top-level (parent=%d)" % (task_id, parent))
-            # otherwise, find the top-level task and go from there
-            seen = {task_id: 1}
-            while parent is not None:
-                if parent in seen:
-                    raise koji.GenericError("Task LOOP at task %i" % task_id)
-                task_id = parent
-                seen[task_id] = 1
-                query.values = {'task_id': task_id}
-                parent = query.singleValue()
-            return Task(task_id).cancelFull(strict=True)
-        # We handle the recursion ourselves, since self.cancel will stop at
-        # canceled or closed tasks.
-        tasklist = [task_id]
-        seen = {}
-        # query for use in loop
-        for task_id in tasklist:
-            if task_id in seen:
-                # shouldn't happen
-                raise koji.GenericError("Task LOOP at task %i" % task_id)
-            seen[task_id] = 1
-            Task(task_id).cancel(recurse=False)
-            query = QueryProcessor(tables=['task'], columns=['id'],
-                                   clauses=['parent = %(task_id)i'],
-                                   values={'task_id': task_id}, opts={'asList': True})
-            for (child_id,) in query.execute():
-                tasklist.append(child_id)
-
-    def getRequest(self):
-        query = QueryProcessor(columns=['request'], tables=['task'],
-                               clauses=['id = %(id)i'], values={'id': self.id})
-        xml_request = query.singleValue()
-        if xml_request.find('<?xml', 0, 10) == -1:
-            # handle older base64 encoded data
-            xml_request = base64.b64decode(xml_request)
-        # note: loads accepts either bytes or string
-        params, method = xmlrpc.client.loads(xml_request)
-        return params
-
-    def getResult(self, raise_fault=True):
-        query = QueryProcessor(tables=['task'], columns=['state', 'result'],
-                               clauses=['id = %(id)i'], values={'id': self.id})
-        r = query.executeOne()
-        if not r:
-            raise koji.GenericError("No such task")
-        state = r['state']
-        xml_result = r['result']
-        if koji.TASK_STATES[state] == 'CANCELED':
-            raise koji.GenericError("Task %i is canceled" % self.id)
-        elif koji.TASK_STATES[state] not in ['CLOSED', 'FAILED']:
-            raise koji.GenericError("Task %i is not finished" % self.id)
-        if xml_result.find('<?xml', 0, 10) == -1:
-            # handle older base64 encoded data
-            xml_result = base64.b64decode(xml_result)
-        try:
-            # If the result is a Fault, then loads will raise it
-            # This is normally what we want to happen
-            result, method = xmlrpc.client.loads(xml_result)
-        except xmlrpc.client.Fault as fault:
-            if raise_fault:
-                raise
-            # Note that you can't really return a fault over xmlrpc, except by
-            # raising it. We return a dictionary in the same format that
-            # multiCall does.
-            return {'faultCode': fault.faultCode, 'faultString': fault.faultString}
-        return result[0]
-
-    def getInfo(self, strict=True, request=False):
-        """Return information about the task in a dictionary.  If "request" is True,
-        the request will be decoded and included in the dictionary."""
-        columns, aliases = self._split_fields()
-        query = QueryProcessor(columns=columns, aliases=aliases,
-                               tables=['task'], clauses=['id = %(id)i'],
-                               values={'id': self.id})
-        result = query.executeOne(strict=strict)
-        if result and request:
-            result['request'] = self.getRequest()
-        return result
-
-    def getChildren(self, request=False):
-        """Return information about tasks with this task as their
-        parent.  If there are no such Tasks, return an empty list."""
-        fields = self.fields
-        if request:
-            fields = fields + (('request', 'request'),)
-        columns, aliases = self._split_fields(fields)
-        query = QueryProcessor(columns=columns, aliases=aliases,
-                               tables=['task'], clauses=['parent = %(id)i'],
-                               values={'id': self.id})
-        results = query.execute()
-        if request:
-            for task in results:
-                if task['request'].find('<?xml', 0, 10) == -1:
-                    # handle older base64 encoded data
-                    task['request'] = base64.b64decode(task['request'])
-                # note: loads accepts either bytes or string
-                task['request'] = xmlrpc.client.loads(task['request'])[0]
-        return results
-
-    def runCallbacks(self, cbtype, old_info, attr, new_val):
-        if cbtype.startswith('pre'):
-            info = old_info
-        elif cbtype.startswith('post'):
-            info = self.getInfo(request=True)
-            if info['state'] == koji.TASK_STATES['CLOSED']:
-                # if task is closed, include the result as well
-                info['result'] = self.getResult()
-            new_val = info[attr]
-        else:
-            raise koji.GenericError('No such callback type: %s' % cbtype)
-        old_val = old_info[attr]
-        if attr == 'state':
-            # state is passed in as an integer, but we want to use the string
-            old_val = koji.TASK_STATES[old_val]
-            new_val = koji.TASK_STATES[new_val]
-        koji.plugin.run_callbacks(cbtype, attribute=attr, old=old_val, new=new_val,
-                                  info=info)

From 67a17401aaf6282a6af432ae019e77f6fff2b4cf Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 41/92] do some task freeing


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index ea98229..e427209 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -3,6 +3,7 @@ import psycopg2
 import time
 
 import koji
+from . import kojihub
 from .db import QueryProcessor, InsertProcessor, UpdateProcessor, db_lock
 from .util import convert_value
 from koji.context import context
@@ -62,8 +63,8 @@ def getTaskRuns(taskID=None, hostID=None, active=None):
         ('scheduler_task_runs.id', 'id'),
         ('scheduler_task_runs.task_id', 'task_id'),
         ('scheduler_task_runs.host_id', 'host_id'),
-    #    ('host.name', 'host_name'),
-    #    ('task.method', 'method'),
+        # ('host.name', 'host_name'),
+        # ('task.method', 'method'),
         ('scheduler_task_runs.active', 'active'),
         ("date_part('epoch', scheduler_task_runs.create_time)", 'create_ts'),
     )
@@ -214,7 +215,7 @@ class TaskScheduler(object):
                 taskruns = runs.get(task['task_id'], [])
                 if not taskruns:
                     logger.error('No active run for assigned task %(task_id)s', task)
-                    # TODO free
+                    kojihub.Task(task['task_id']).free()
                     continue
                 else:
                     if len(taskruns) > 1:
@@ -223,19 +224,21 @@ class TaskScheduler(object):
                         # TODO fix
                     age = time.time() - min([r['create_ts'] for r in taskruns])
                     if age > self.assign_timeout:
-                        # TODO free
                         # TODO check host too
                         logger.info('Task assignment timeout for %(task_id)s', task)
+                        kojihub.Task(task['task_id']).free()
                         pass
             elif task['state'] == koji.TASK_STATES['OPEN']:
                 # TODO sanity check host
                 if not task['host_id']:
                     # shouldn't happen
-                    # TODO
+                    logger.error('Open task with no host %(task_id)s', task)
+                    kojihub.Task(task['task_id']).free()
                     continue
                 host = self.hosts.get(task['host_id'])
                 if not host:
                     logger.error('Host for task is not available')
+                    # TODO
 
     def get_active_runs(self):
         runs = getTaskRuns(active=True)

From ebd8eae58c96c24bf595222d69b7a3ce46d2b432 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 42/92] export getTaskRuns


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index e427209..6229c62 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -17,10 +17,6 @@ class DBLogger:
     pass
 
 
-class SchedulerExports:
-    pass
-
-
 def intlist(value):
     """Cast value to a list of ints"""
     if isinstance(value, (list, tuple)):
@@ -54,7 +50,7 @@ def get_tasks_for_host(hostID):
     return query.execute()
 
 
-def getTaskRuns(taskID=None, hostID=None, active=None):
+def get_task_runs(taskID=None, hostID=None, active=None):
     taskID = convert_value(taskID, cast=int, none_allowed=True)
     hostID = convert_value(hostID, cast=int, none_allowed=True)
     active = convert_value(active, cast=bool, none_allowed=True)
@@ -241,7 +237,7 @@ class TaskScheduler(object):
                     # TODO
 
     def get_active_runs(self):
-        runs = getTaskRuns(active=True)
+        runs = get_task_runs(active=True)
         runs_by_task = {}
         for run in runs:
             runs_by_task.setdefault(run['task_id'], [])
@@ -391,3 +387,7 @@ class TaskScheduler(object):
                 values={'task_id': task['task_id'], 'free': koji.TASK_STATES['FREE']},
         )
         update.execute()
+
+
+class SchedulerExports:
+    getTaskRuns = staticmethod(get_task_runs)

From 95aa93682ca46ab8e411fd8c39d611ac16d8edf6 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 43/92] end stale runs


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 6229c62..97923a6 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -236,6 +236,17 @@ class TaskScheduler(object):
                     logger.error('Host for task is not available')
                     # TODO
 
+        # end stale runs
+        update = UpdateProcessor(
+                'scheduler_task_runs',
+                data={'active': False},
+                clauses=['active = TRUE',
+                         '(SELECT id FROM task WHERE task.id=task_id AND state IN %(states)s) IS NULL'],
+                values={'states': [koji.TASK_STATES[s] for s in ('OPEN', 'ASSIGNED')]},
+        )
+        update.execute()
+
+
     def get_active_runs(self):
         runs = get_task_runs(active=True)
         runs_by_task = {}

From 4b5510f0a1c42013f1722b6f8ad0a0d260744f87 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 44/92] implement host timeout in scheduler


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 97923a6..81cb88e 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -104,6 +104,7 @@ class TaskScheduler(object):
         self.maxjobs = 15  # XXX
         self.capacity_overcommit = 5
         self.assign_timeout = 300
+        self.host_timeout = 900
 
     def run(self):
         if not db_lock('scheduler', wait=False):
@@ -203,38 +204,50 @@ class TaskScheduler(object):
         logger.info('Found %i active runs', len(runs))
         logger.info('Checking on %i active tasks', len(self.active_tasks))
         for task in self.active_tasks:
+
+            if not task['host_id']:
+                logger.error('Active task with no host: %s', task['task_id'])
+                kojihub.Task(task['task_id']).free()
+                continue
+
+            host = self.hosts.get(task['host_id'])
+            if not host:
+                # host disabled?
+                # TODO
+                continue
+
+            taskruns = runs.get(task['task_id'], [])
+            if not taskruns:
+                logger.error('No active run for assigned task %(task_id)s', task)
+                kojihub.Task(task['task_id']).free()
+                continue
+
+            if len(taskruns) > 1:
+                logger.error('Multiple active run entries for assigned task %(task_id)s',
+                             task)
+                # TODO fix
+
             if task['state'] == koji.TASK_STATES['ASSIGNED']:
                 # TODO check time since assigned
                 # if not taken within a timeout
                 #  - if host not checking in, then make sure host marked unavail and free
                 #  - if host *is* checking in, then treat as refusal and free
-                taskruns = runs.get(task['task_id'], [])
-                if not taskruns:
-                    logger.error('No active run for assigned task %(task_id)s', task)
+                age = time.time() - min([r['create_ts'] for r in taskruns])
+                if age > self.assign_timeout:
+                    logger.info('Task assignment timeout for %(task_id)s', task)
                     kojihub.Task(task['task_id']).free()
-                    continue
-                else:
-                    if len(taskruns) > 1:
-                        logger.error('Multiple active run entries for assigned task %(task_id)s',
-                                     task)
-                        # TODO fix
-                    age = time.time() - min([r['create_ts'] for r in taskruns])
-                    if age > self.assign_timeout:
-                        # TODO check host too
-                        logger.info('Task assignment timeout for %(task_id)s', task)
-                        kojihub.Task(task['task_id']).free()
-                        pass
+
             elif task['state'] == koji.TASK_STATES['OPEN']:
-                # TODO sanity check host
-                if not task['host_id']:
-                    # shouldn't happen
-                    logger.error('Open task with no host %(task_id)s', task)
+                if host['update_ts'] is None:
+                    # shouldn't happen?
+                    # fall back to task_run time
+                    age = time.time() - min([r['create_ts'] for r in taskruns])
+                else:
+                    age = time.time() - host['update_ts']
+                if age > self.host_timeout:
+                    logger.info('Freeing task %s from unresponsive host %s',
+                                task['task_id'], host['name'])
                     kojihub.Task(task['task_id']).free()
-                    continue
-                host = self.hosts.get(task['host_id'])
-                if not host:
-                    logger.error('Host for task is not available')
-                    # TODO
 
         # end stale runs
         update = UpdateProcessor(
@@ -330,10 +343,10 @@ class TaskScheduler(object):
         fields = (
             ('host.id', 'id'),
             ('host.name', 'name'),
+            ("date_part('epoch', host.update_time)", 'update_ts'),
             ('host.task_load', 'task_load'),
             ('host_config.arches', 'arches'),
             ('host_config.capacity', 'capacity'),
-        #    ("date_part('epoch', sessions.update_time)", 'update_ts'),
         )
         fields, aliases = zip(*fields)
 
@@ -345,12 +358,8 @@ class TaskScheduler(object):
 #                'host.ready IS TRUE',
                 'host_config.enabled IS TRUE',
                 'host_config.active IS TRUE',
-#                'sessions.expired IS FALSE',
-#                'sessions.master IS NULL',
-#                "sessions.update_time > NOW() - '5 minutes'::interval"
             ],
             joins=[
-            #    'sessions USING (user_id)',
                 'host_config ON host.id = host_config.host_id'
             ]
         )

From 1d878594e8615c49e6e1998b09f32420df493dbd Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 45/92] fix stale ready states


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 81cb88e..95f47f5 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -103,6 +103,7 @@ class TaskScheduler(object):
         # TODO these things need proper config
         self.maxjobs = 15  # XXX
         self.capacity_overcommit = 5
+        self.ready_timeout = 180
         self.assign_timeout = 300
         self.host_timeout = 900
 
@@ -112,8 +113,8 @@ class TaskScheduler(object):
             return False
 
         self.do_schedule()
-        # TODO clean up bad data (e.g. active tasks with no host)
         self.check_active_tasks()
+        self.check_hosts()
 
         return True
 
@@ -260,6 +261,25 @@ class TaskScheduler(object):
         update.execute()
 
 
+    def check_hosts(self):
+        # sanity check ready status
+        hosts_to_mark = []
+        for host in self.hosts.values():
+            if not host['ready']:
+                continue
+            if (host['update_ts'] is None or time.time() - host['update_ts'] > self.ready_timeout):
+                hosts_to_mark.append(host)
+
+        if hosts_to_mark:
+            update = db.UpdateProcessor(
+                    'host',
+                    data={'ready': False},
+                    clauses=['host_id IN %(host_ids)s'],
+                    values={'host_ids': [h['id'] for h in hosts_to_mark]},
+            )
+            update.execute()
+
+
     def get_active_runs(self):
         runs = get_task_runs(active=True)
         runs_by_task = {}
@@ -325,7 +345,7 @@ class TaskScheduler(object):
         # get hosts and bin them
         hosts_by_bin = {}
         hosts_by_id = {}
-        for host in self.get_ready_hosts():
+        for host in self._get_hosts():
             host['_bins'] = []
             hosts_by_id[host['id']] = host
             for chan in host['channels']:
@@ -337,14 +357,15 @@ class TaskScheduler(object):
         self.hosts_by_bin = hosts_by_bin
         self.hosts = hosts_by_id
 
-    def get_ready_hosts(self):
-        """Query hosts that are ready to build"""
+    def _get_hosts(self):
+        """Query enabled hosts"""
 
         fields = (
             ('host.id', 'id'),
             ('host.name', 'name'),
             ("date_part('epoch', host.update_time)", 'update_ts'),
             ('host.task_load', 'task_load'),
+            ('host.ready', 'ready'),
             ('host_config.arches', 'arches'),
             ('host_config.capacity', 'capacity'),
         )
@@ -355,7 +376,6 @@ class TaskScheduler(object):
             columns=fields,
             aliases=aliases,
             clauses=[
-#                'host.ready IS TRUE',
                 'host_config.enabled IS TRUE',
                 'host_config.active IS TRUE',
             ],

From 143893fb8da7bf633d67d5623b08322264d2717e Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 46/92] simple db logging


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 016bb11..39bcf56 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -97,7 +97,6 @@ from .util import convert_value
 
 
 logger = logging.getLogger('koji.hub')
-sched_logger = scheduler.DBLogger()
 
 
 NUMERIC_TYPES = (int, float)
@@ -14355,8 +14354,6 @@ class HostExports(object):
             insert = InsertProcessor(table=table, data={'data': hostdata},
                                      clauses=clauses, values=values)
             insert.execute()
-        sched_logger.debug(f"Updating host data with: {hostdata}",
-                           host_id=host.id, location='setHostData')
 
     def getTasks(self):
         host = Host()
@@ -14377,9 +14374,6 @@ class HostExports(object):
             }
         )
         tasks = query.execute()
-        for task in tasks:
-            sched_logger.debug("Sending task", host_id=host.id, task_id=task['id'],
-                               location="getTasks")
         return tasks
 
     def refuseTask(self, task_id):
@@ -14390,8 +14384,6 @@ class HostExports(object):
 
         task = Task(task_id)
         task.free(newstate=koji.TASK_STATES['REFUSED'])
-        sched_logger.warning("Refusing task", host_id=host.id, task_id=task_id,
-                             location="refuseTask")
         return True
 
     def getHostTasks(self):
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 95f47f5..881d49b 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -10,11 +10,21 @@ from koji.context import context
 
 
 logger = logging.getLogger('koji.scheduler')
-# TODO set up db logging
 
 
-class DBLogger:
-    pass
+def log_db(msg, task_id=None, host_id=None):
+    insert = InsertProcessor(
+            'scheduler_log_messages',
+            data={'msg': msg, 'task_id': task_id, 'host_id': host_id},
+    )
+    insert.execute()
+
+
+def log_both(msg, task_id=None, host_id=None, level=logging.INFO):
+    pre1 = f"[task_id={task_id}] " if task_id else ""
+    pre2 = f"[host_id={host_id}] " if host_id else ""
+    logger.log(level, '%s%s%s', pre1, pre2, msg)
+    log_db(msg, task_id, host_id)
 
 
 def intlist(value):
@@ -207,7 +217,7 @@ class TaskScheduler(object):
         for task in self.active_tasks:
 
             if not task['host_id']:
-                logger.error('Active task with no host: %s', task['task_id'])
+                log_both('Active task with no host', task_id=task['task_id'], level=logging.ERROR)
                 kojihub.Task(task['task_id']).free()
                 continue
 
@@ -219,7 +229,8 @@ class TaskScheduler(object):
 
             taskruns = runs.get(task['task_id'], [])
             if not taskruns:
-                logger.error('No active run for assigned task %(task_id)s', task)
+                log_both('Assigned task with no active run entry', task_id=task['task_id'],
+                         host_id=host['id'], level=logging.ERROR)
                 kojihub.Task(task['task_id']).free()
                 continue
 
@@ -235,7 +246,8 @@ class TaskScheduler(object):
                 #  - if host *is* checking in, then treat as refusal and free
                 age = time.time() - min([r['create_ts'] for r in taskruns])
                 if age > self.assign_timeout:
-                    logger.info('Task assignment timeout for %(task_id)s', task)
+                    log_both('Task assignment timeout', task_id=task['task_id'],
+                             host_id=host['id'])
                     kojihub.Task(task['task_id']).free()
 
             elif task['state'] == koji.TASK_STATES['OPEN']:
@@ -246,8 +258,8 @@ class TaskScheduler(object):
                 else:
                     age = time.time() - host['update_ts']
                 if age > self.host_timeout:
-                    logger.info('Freeing task %s from unresponsive host %s',
-                                task['task_id'], host['name'])
+                    log_both('Freeing task from unresponsive host', task_id=task['task_id'],
+                             host_id=host['id'])
                     kojihub.Task(task['task_id']).free()
 
         # end stale runs
@@ -269,6 +281,7 @@ class TaskScheduler(object):
                 continue
             if (host['update_ts'] is None or time.time() - host['update_ts'] > self.ready_timeout):
                 hosts_to_mark.append(host)
+                log_both('Marking host not ready', host_id=host['id'])
 
         if hosts_to_mark:
             update = db.UpdateProcessor(
@@ -402,8 +415,7 @@ class TaskScheduler(object):
         return hosts
 
     def add_run(self, task, host):
-        logger.info('Assigning task %s (%s) to host %s',
-                    task['task_id'], task['method'], host['name'])
+        log_both('Assigning task', task_id=task['task_id'], host_id=host['id'])
 
         # mark any older runs inactive
         update = UpdateProcessor(
diff --git a/schemas/schema.sql b/schemas/schema.sql
index 57ff6f5..9aae6bf 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -1026,9 +1026,6 @@ CREATE TABLE scheduler_log_messages (
         task_id INTEGER REFERENCES task (id),
         host_id INTEGER REFERENCES host (id),
         msg_time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-        logger_name VARCHAR(200) NOT NULL,
-        level VARCHAR(10) NOT NULL,
-        location VARCHAR(200),
         msg TEXT NOT NULL
 ) WITHOUT OIDS;
 

From 7209264c0dcb31b7a151b07cf5c19cdd93f50d4e Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 47/92] fixes


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 881d49b..07c3bc5 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -122,16 +122,15 @@ class TaskScheduler(object):
             # already running elsewhere
             return False
 
+        self.get_tasks()
+        self.get_hosts()
+        self.check_hosts()
         self.do_schedule()
         self.check_active_tasks()
-        self.check_hosts()
 
         return True
 
     def do_schedule(self):
-        self.get_tasks()
-        self.get_hosts()
-
         # debug
         logger.info('Running task scheduler')
         logger.info(f'Hosts: {len(self.hosts)}')
@@ -169,8 +168,8 @@ class TaskScheduler(object):
             task['_hosts'] = []
             min_avail = min(0, task['weight'] - self.capacity_overcommit)
             for host in self.hosts_by_bin.get(task['_bin'], []):
-                if (host['capacity'] - host['_load'] > min_avail and
-                        host['_ntasks'] < self.maxjobs):
+                if (host['ready'] and host['_ntasks'] < self.maxjobs and
+                        host['capacity'] - host['_load'] > min_avail):
                     task['_hosts'].append(host)
             logger.info(f'Task {task["task_id"]}: {len(task["_hosts"])} options')
             #import pdb; pdb.set_trace()

From aa52000a466c3e4d680a69115c43e266576f1d16 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 48/92] watch-tasks: show host for assigned tasks


---

diff --git a/cli/koji_cli/lib.py b/cli/koji_cli/lib.py
index 2ec1a1f..c605940 100644
--- a/cli/koji_cli/lib.py
+++ b/cli/koji_cli/lib.py
@@ -261,12 +261,13 @@ class TaskWatcher(object):
         # not finished either.  info would be none.
         if not info:
             return 'unknown'
-        if info['state'] == koji.TASK_STATES['OPEN']:
+        if koji.TASK_STATES[info['state']] in ['OPEN', 'ASSIGNED']:
+            state = koji.TASK_STATES[info['state']].lower()
             if info['host_id']:
                 host = self.session.getHost(info['host_id'])
-                return 'open (%s)' % host['name']
+                return '%s (%s)' % (state, host['name'])
             else:
-                return 'open'
+                return state
         elif info['state'] == koji.TASK_STATES['FAILED']:
             s = 'FAILED: %s' % self.get_failure()
 

From b74ef2dc852dd74546739cd9d1968b216c6431d5 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 49/92] Revert "move convert_value to util lib"


This reverts commit 61f5d2bd46623a9fd9c74ae67f81a7d43da7821f.

---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 39bcf56..fabbd1c 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -93,7 +93,6 @@ from .db import (  # noqa: F401
     nextval,
     currval,
 )
-from .util import convert_value
 
 
 logger = logging.getLogger('koji.hub')
@@ -111,6 +110,36 @@ def xform_user_krb(entry):
     return entry
 
 
+def convert_value(value, cast=None, message=None,
+                  exc_type=koji.ParameterError, none_allowed=False, check_only=False):
+    """Cast to another type with tailored exception
+
+    :param any value: tested object
+    :param type cast: To which type value should be cast
+    :param type exc_type: Raise this exception
+    :param bool none_allowed: Is None valid value?
+    :param check_only: Don't convert but raise an exception if type(value) != cast
+
+    :returns any value: returns converted value
+    """
+    if value is None:
+        if not none_allowed:
+            raise exc_type(message or f"Invalid type, expected type {cast}")
+        else:
+            return value
+    if check_only:
+        if not isinstance(value, cast):
+            raise exc_type(message or f"Invalid type for value '{value}': {type(value)}, "
+                                      f"expected type {cast}")
+    else:
+        try:
+            value = cast(value)
+        except (ValueError, TypeError):
+            raise exc_type(message or f"Invalid type for value '{value}': {type(value)}, "
+                                      f"expected type {cast}")
+    return value
+
+
 class Task(object):
     """A task for the build hosts"""
 
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 07c3bc5..768e27c 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -5,10 +5,12 @@ import time
 import koji
 from . import kojihub
 from .db import QueryProcessor, InsertProcessor, UpdateProcessor, db_lock
-from .util import convert_value
 from koji.context import context
 
 
+convert_value = kojihub.convert_value
+
+
 logger = logging.getLogger('koji.scheduler')
 
 
diff --git a/kojihub/util.py b/kojihub/util.py
deleted file mode 100644
index a874676..0000000
--- a/kojihub/util.py
+++ /dev/null
@@ -1,33 +0,0 @@
-import koji
-
-
-def convert_value(value, cast=None, message=None,
-                  exc_type=koji.ParameterError, none_allowed=False, check_only=False):
-    """Cast to another type with tailored exception
-
-    :param any value: tested object
-    :param type cast: To which type value should be cast
-    :param type exc_type: Raise this exception
-    :param bool none_allowed: Is None valid value?
-    :param check_only: Don't convert but raise an exception if type(value) != cast
-
-    :returns any value: returns converted value
-    """
-    if value is None:
-        if not none_allowed:
-            raise exc_type(message or f"Invalid type, expected type {cast}")
-        else:
-            return value
-    if check_only:
-        if not isinstance(value, cast):
-            raise exc_type(message or f"Invalid type for value '{value}': {type(value)}, "
-                                      f"expected type {cast}")
-    else:
-        try:
-            value = cast(value)
-        except (ValueError, TypeError):
-            raise exc_type(message or f"Invalid type for value '{value}': {type(value)}, "
-                                      f"expected type {cast}")
-    return value
-
-

From b3e28a2a6b2337a2caa088236a1b6a0468174c26 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 50/92] delete old scheduler log messages


---

diff --git a/util/koji-sweep-db b/util/koji-sweep-db
index cd62346..373bb8e 100755
--- a/util/koji-sweep-db
+++ b/util/koji-sweep-db
@@ -147,6 +147,21 @@ def clean_buildroots(cursor, vacuum, test):
             cursor.execute("VACUUM ANALYZE buildroot")
 
 
+def clean_scheduler_logs(cursor, vacuum, test, age):
+    clauses = f"(msg_time < NOW() - '{age:d} days'::interval)"
+    if options.verbose:
+        query = QueryProcessor(tables=["scheduler_log_messages"],
+                               clauses=clauses,
+                               opts={'countOnly': True})
+        rows = query.execute()
+        print(f"Deleting {rows} scheduler log messages")
+    if not test:
+        delete = DeleteProcessor(table="scheduler_log_messages", clauses=clauses)\
+        delete.execute()
+        if vacuum:
+            cursor.execute("VACUUM ANALYZE scheduler_log_messages")
+
+
 if __name__ == "__main__":
     global options
     parser = OptionParser("%prog cleans koji database")
@@ -180,6 +195,9 @@ if __name__ == "__main__":
     parser.add_option('--scratch-builds-age', type=int, dest="scratch_age",
                       action="store", default=730, metavar="DAYS",
                       help="Delete scratch builds' tasks older than this (default: 2 years")
+    parser.add_option('--logs-age', type=int,
+                      action="store", default=7, metavar="DAYS",
+                      help="Delete scheduler log messages older than this (default: 7 days)")
     parser.add_option('--buildroots', action="store_true",
                       help="Delete unreferenced buildroots")
     parser.add_option('-f', '--force', action="store_true",
@@ -240,6 +258,7 @@ if __name__ == "__main__":
     clean_sessions(cursor, options.vacuum, options.test, options.sessions_age,
                    options.sessions_absolute_age)
     clean_reservations(cursor, options.vacuum, options.test, options.reservations_age)
+    clean_scheduler_logs(cursor, options.vacuum, options.test, options.logs_age)
     if options.tag_notifications:
         clean_notification_tasks(cursor, options.vacuum, options.test,
                                  age=options.tag_notifications_age)

From 69c9410496b0cc105803f2d7bf445803fa2ffa49 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 51/92] flake8


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 768e27c..df1be0c 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -1,11 +1,9 @@
 import logging
-import psycopg2
 import time
 
 import koji
 from . import kojihub
 from .db import QueryProcessor, InsertProcessor, UpdateProcessor, db_lock
-from koji.context import context
 
 
 convert_value = kojihub.convert_value
@@ -16,8 +14,8 @@ logger = logging.getLogger('koji.scheduler')
 
 def log_db(msg, task_id=None, host_id=None):
     insert = InsertProcessor(
-            'scheduler_log_messages',
-            data={'msg': msg, 'task_id': task_id, 'host_id': host_id},
+        'scheduler_log_messages',
+        data={'msg': msg, 'task_id': task_id, 'host_id': host_id},
     )
     insert.execute()
 
@@ -88,21 +86,12 @@ def get_task_runs(taskID=None, hostID=None, active=None):
 
     query = QueryProcessor(
         columns=fields, aliases=aliases, tables=['scheduler_task_runs'],
-    #    joins=['host ON host_id=host.id', 'task ON task_id=task.id'],
+        # joins=['host ON host_id=host.id', 'task ON task_id=task.id'],
         clauses=clauses, values=locals())
 
     return query.execute()
 
 
-def scheduler_map_task(taskinfo):
-    # map which hosts can take this task
-    # eventually this will involve more complex rules
-    q = QueryProcessor()
-    # select hosts matching arch and channel
-    hosts = q.execute()
-    u = InsertProcessor()
-
-
 class TaskScheduler(object):
 
     def __init__(self):
@@ -174,7 +163,6 @@ class TaskScheduler(object):
                         host['capacity'] - host['_load'] > min_avail):
                     task['_hosts'].append(host)
             logger.info(f'Task {task["task_id"]}: {len(task["_hosts"])} options')
-            #import pdb; pdb.set_trace()
             for host in task['_hosts']:
                 # demand gives us a rough measure of how much overall load is pending for the host
                 host.setdefault('_demand', 0.0)
@@ -193,7 +181,8 @@ class TaskScheduler(object):
         for task in self.free_tasks:
             min_avail = task['weight'] - self.capacity_overcommit
             task['_hosts'].sort(key=lambda h: h['_rank'])
-            logger.debug('Task %i choices: %s', task['task_id'], [(h['name'], "%(_rank).2f" %h) for h in task['_hosts']])
+            logger.debug('Task %i choices: %s', task['task_id'],
+                         [(h['name'], "%(_rank).2f" % h) for h in task['_hosts']])
             for host in task['_hosts']:
                 if (host['capacity'] - host['_load'] > min_avail and
                         host['_ntasks'] < self.maxjobs):
@@ -265,15 +254,15 @@ class TaskScheduler(object):
 
         # end stale runs
         update = UpdateProcessor(
-                'scheduler_task_runs',
-                data={'active': False},
-                clauses=['active = TRUE',
-                         '(SELECT id FROM task WHERE task.id=task_id AND state IN %(states)s) IS NULL'],
-                values={'states': [koji.TASK_STATES[s] for s in ('OPEN', 'ASSIGNED')]},
+            'scheduler_task_runs',
+            data={'active': False},
+            clauses=['active = TRUE',
+                     '(SELECT id FROM task WHERE task.id=task_id AND '
+                     'state IN %(states)s) IS NULL'],
+            values={'states': [koji.TASK_STATES[s] for s in ('OPEN', 'ASSIGNED')]},
         )
         update.execute()
 
-
     def check_hosts(self):
         # sanity check ready status
         hosts_to_mark = []
@@ -285,15 +274,14 @@ class TaskScheduler(object):
                 log_both('Marking host not ready', host_id=host['id'])
 
         if hosts_to_mark:
-            update = db.UpdateProcessor(
-                    'host',
-                    data={'ready': False},
-                    clauses=['host_id IN %(host_ids)s'],
-                    values={'host_ids': [h['id'] for h in hosts_to_mark]},
+            update = UpdateProcessor(
+                'host',
+                data={'ready': False},
+                clauses=['host_id IN %(host_ids)s'],
+                values={'host_ids': [h['id'] for h in hosts_to_mark]},
             )
             update.execute()
 
-
     def get_active_runs(self):
         runs = get_task_runs(active=True)
         runs_by_task = {}
@@ -327,7 +315,7 @@ class TaskScheduler(object):
             columns=fields, aliases=aliases, tables=['task'],
             clauses=('task.state IN %(states)s',
                      'task.host_id IS NOT NULL',  # should always be set, but...
-                    ),
+                     ),
             values=values,
         )
         active_tasks = query.execute()
@@ -420,10 +408,10 @@ class TaskScheduler(object):
 
         # mark any older runs inactive
         update = UpdateProcessor(
-                'scheduler_task_runs',
-                data={'active': False},
-                clauses=['task_id=%(task_id)s', 'active = TRUE'],
-                values={'task_id': task['task_id']},
+            'scheduler_task_runs',
+            data={'active': False},
+            clauses=['task_id=%(task_id)s', 'active = TRUE'],
+            values={'task_id': task['task_id']},
         )
         update.execute()
 
@@ -434,10 +422,10 @@ class TaskScheduler(object):
 
         # mark the task assigned
         update = UpdateProcessor(
-                'task',
-                data={'host_id': host['id'], 'state': koji.TASK_STATES['ASSIGNED']},
-                clauses=['id=%(task_id)s', 'state=%(free)s'],
-                values={'task_id': task['task_id'], 'free': koji.TASK_STATES['FREE']},
+            'task',
+            data={'host_id': host['id'], 'state': koji.TASK_STATES['ASSIGNED']},
+            clauses=['id=%(task_id)s', 'state=%(free)s'],
+            values={'task_id': task['task_id'], 'free': koji.TASK_STATES['FREE']},
         )
         update.execute()
 

From ef6144b0b8c1382e2ddaa640ecbd57311799fff6 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 52/92] pull getHostData() from pr3631


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index df1be0c..2fc51d2 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -60,6 +60,27 @@ def get_tasks_for_host(hostID):
     return query.execute()
 
 
+def get_host_data(hostID=None):
+    """Return actual builder data
+
+    :param int hostID: Return data for given host (otherwise for all)
+    :returns list[dict]: list of host_id/data dicts
+    """
+    clauses = []
+    columns = ['host_id', 'data']
+    if hostID is not None:
+        clauses.append('host_id = %(hostID)i')
+    query = QueryProcessor(
+        tables=['scheduler_host_data'],
+        clauses=clauses,
+        columns=columns,
+        values=locals(),
+        opts={'order': 'host_id'}
+    )
+
+    return query.execute()
+
+
 def get_task_runs(taskID=None, hostID=None, active=None):
     taskID = convert_value(taskID, cast=int, none_allowed=True)
     hostID = convert_value(hostID, cast=int, none_allowed=True)
@@ -432,3 +453,4 @@ class TaskScheduler(object):
 
 class SchedulerExports:
     getTaskRuns = staticmethod(get_task_runs)
+    getHostData = staticmethod(get_host_data)

From e0dfc40dbb42c25d92b26f894d4b4b9cb687f950 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 53/92] fix convert_value refs


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 2fc51d2..600acfc 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -6,9 +6,6 @@ from . import kojihub
 from .db import QueryProcessor, InsertProcessor, UpdateProcessor, db_lock
 
 
-convert_value = kojihub.convert_value
-
-
 logger = logging.getLogger('koji.scheduler')
 
 
@@ -37,7 +34,7 @@ def intlist(value):
 
 def get_tasks_for_host(hostID):
     """Get the tasks assigned to a given host"""
-    hostID = convert_value(hostID, cast=int, none_allowed=True)
+    hostID = kojihub.convert_value(hostID, cast=int, none_allowed=True)
 
     fields = (
         ('task.id', 'id'),
@@ -82,9 +79,9 @@ def get_host_data(hostID=None):
 
 
 def get_task_runs(taskID=None, hostID=None, active=None):
-    taskID = convert_value(taskID, cast=int, none_allowed=True)
-    hostID = convert_value(hostID, cast=int, none_allowed=True)
-    active = convert_value(active, cast=bool, none_allowed=True)
+    taskID = kojihub.convert_value(taskID, cast=int, none_allowed=True)
+    hostID = kojihub.convert_value(hostID, cast=int, none_allowed=True)
+    active = kojihub.convert_value(active, cast=bool, none_allowed=True)
 
     fields = (
         ('scheduler_task_runs.id', 'id'),

From a76c2762128ff794ee1cf0d503ceca66cfb88761 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 54/92] drop unused var


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 600acfc..90b62f9 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -115,7 +115,6 @@ class TaskScheduler(object):
     def __init__(self):
         self.hosts_by_bin = None
         self.hosts = None
-        self.tasks_by_bin = None
         self.active_tasks = None
         self.free_tasks = None
 

From fabdaab6aca43d31aca23e31ed31dcfd43225025 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 55/92] scheduler check_ts fragment


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 90b62f9..c7ad458 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -1,3 +1,4 @@
+import json
 import logging
 import time
 
@@ -124,12 +125,17 @@ class TaskScheduler(object):
         self.ready_timeout = 180
         self.assign_timeout = 300
         self.host_timeout = 900
+        self.run_interval = 60
 
     def run(self):
         if not db_lock('scheduler', wait=False):
             # already running elsewhere
             return False
 
+        if not self.check_ts():
+            # already ran too recently
+            return False
+
         self.get_tasks()
         self.get_hosts()
         self.check_hosts()
@@ -138,6 +144,47 @@ class TaskScheduler(object):
 
         return True
 
+    def check_ts(self):
+        """Check the last run timestamp
+
+        Returns True if the scheduler should run, False otherwise
+        """
+
+        # get last ts
+        query = QueryProcessor(
+            tables=['scheduler_sys_data'],
+            columns=['data'],
+            clauses=['name = %(name)s'],
+            values={'name': 'last_run_ts'},
+        )
+        last = query.singleValue(strict=False) or 0
+
+        now = time.time()
+        delta = now - last
+
+        if delta < 0:
+            logger.error('Last run in the future by %i seconds', -delta)
+            ret = False
+            # update the ts so that a system time rollback doesn't keep us from running
+        elif delta < self.run_interval:
+            logger.debug('Skipping run due to run_interval setting')
+            # return now without updating ts
+            return False
+        else:
+            ret = True
+
+        # save current ts
+        # XXX need an UPSERT
+        update = UpdateProcessor(
+            'scheduler_sys_data',
+            clauses=['name = %(name)s'],
+            values={'name': 'last_run_ts'},
+            data={'data': json.dumps(now)},
+        )
+        update.execute()
+
+        return ret
+
     def do_schedule(self):
         # debug
         logger.info('Running task scheduler')
@@ -294,7 +341,7 @@ class TaskScheduler(object):
             update = UpdateProcessor(
                 'host',
                 data={'ready': False},
-                clauses=['host_id IN %(host_ids)s'],
+                clauses=['id IN %(host_ids)s'],
                 values={'host_ids': [h['id'] for h in hosts_to_mark]},
             )
             update.execute()
diff --git a/schemas/schema.sql b/schemas/schema.sql
index 9aae6bf..a1f2a5b 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -1010,6 +1010,12 @@ CREATE TABLE scheduler_host_data (
 ) WITHOUT OIDS;
 
 
+CREATE TABLE scheduler_sys_data (
+        name TEXT NOT NULL PRIMARY KEY,
+        data JSONB
+) WITHOUT OIDS;
+
+
 CREATE TABLE scheduler_map (
         id SERIAL NOT NULL PRIMARY KEY,
         task_id INTEGER REFERENCES task (id) NOT NULL,

From 451e452d178a4500e5c28258da08e39d37b14726 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 56/92] handle first time case for last_run_ts


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index c7ad458..0d7f928 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -174,14 +174,20 @@ class TaskScheduler(object):
             ret = True
 
         # save current ts
-        # XXX need an UPSERT
         update = UpdateProcessor(
             'scheduler_sys_data',
             clauses=['name = %(name)s'],
             values={'name': 'last_run_ts'},
             data={'data': json.dumps(now)},
         )
-        update.execute()
+        chk = update.execute()
+        if not chk:
+            # hasn't been defined yet
+            insert = InsertProcessor(
+                'scheduler_sys_data',
+                data={'name': 'last_run_ts', 'data': json.dumps(now)},
+            )
+            insert.execute()
 
         return ret
 

From 0867cd32c459ca06957bd257129b95ad75b8ccd6 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 57/92] ...


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 0d7f928..68d17dc 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -136,6 +136,7 @@ class TaskScheduler(object):
             # already ran too recently
             return False
 
+        logger.info('Running task scheduler')
         self.get_tasks()
         self.get_hosts()
         self.check_hosts()
@@ -193,7 +194,6 @@ class TaskScheduler(object):
 
     def do_schedule(self):
         # debug
-        logger.info('Running task scheduler')
         logger.info(f'Hosts: {len(self.hosts)}')
         logger.info(f'Free tasks: {len(self.free_tasks)}')
         logger.info(f'Active tasks: {len(self.active_tasks)}')
diff --git a/schemas/schema.sql b/schemas/schema.sql
index a1f2a5b..6863da0 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -1027,6 +1027,16 @@ CREATE TABLE scheduler_map (
 ) WITHOUT OIDS;
 
 
+CREATE TABLE scheduler_task_refusals (
+        id SERIAL NOT NULL PRIMARY KEY,
+        task_id INTEGER REFERENCES task (id) NOT NULL,
+        host_id INTEGER REFERENCES host (id) NOT NULL,
+        by_host BOOLEAN NOT NULL,
+        msg TEXT,
+        time TIMESTAMPTZ NOT NULL DEFAULT NOW()
+) WITHOUT OIDS;
+
+
 CREATE TABLE scheduler_log_messages (
         id SERIAL NOT NULL PRIMARY KEY,
         task_id INTEGER REFERENCES task (id),

From bc1877cd63ddecb533d82ada706fd178f43ce969 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 58/92] initial task refusal functions


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 68d17dc..0283c26 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -58,6 +58,52 @@ def get_tasks_for_host(hostID):
     return query.execute()
 
 
+def set_refusal(hostID, taskID, soft=True, by_host=False, msg=''):
+    data = {
+        'task_id': kojihub.convert_value(hostID, cast=int),
+        'host_id': kojihub.convert_value(taskID, cast=int),
+        'soft': kojihub.convert_value(soft, cast=bool),
+        'by_host': kojihub.convert_value(by_host, cast=bool),
+        'msg': kojihub.convert_value(msg, cast=str),
+    }
+    insert = InsertProcessor('scheduler_task_refusals', data=data)
+    insert.execute()
+    # note: db allows multiple entries here, but in general we shouldn't
+    # make very many
+
+
+def get_task_refusals(taskID=None, hostID=None):
+    taskID = kojihub.convert_value(taskID, cast=int, none_allowed=True)
+    hostID = kojihub.convert_value(hostID, cast=int, none_allowed=True)
+
+    fields = (
+        ('scheduler_task_refusals.id', 'id'),
+        ('scheduler_task_refusals.task_id', 'task_id'),
+        ('scheduler_task_refusals.host_id', 'host_id'),
+        ('scheduler_task_refusals.by_host', 'by_host'),
+        ('scheduler_task_refusals.soft', 'soft'),
+        ('scheduler_task_refusals.msg', 'msg'),
+        # ('host.name', 'host_name'),
+        ("date_part('epoch', scheduler_task_refusals.time)", 'ts'),
+    )
+    fields, aliases = zip(*fields)
+
+    clauses = []
+    if taskID is not None:
+        clauses.append('task_id = %(taskID)s')
+    if hostID is not None:
+        clauses.append('host_id = %(hostID)s')
+
+    query = QueryProcessor(
+        columns=fields, aliases=aliases, tables=['scheduler_task_refusals'],
+        # joins=['host ON host_id=host.id', 'task ON task_id=task.id'],
+        clauses=clauses, values=locals(),
+        opts={'order': '-id'}
+    )
+
+    return query.execute()
+
+
 def get_host_data(hostID=None):
     """Return actual builder data
 
diff --git a/schemas/schema.sql b/schemas/schema.sql
index 6863da0..de97cf6 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -1032,6 +1032,7 @@ CREATE TABLE scheduler_task_refusals (
         task_id INTEGER REFERENCES task (id) NOT NULL,
         host_id INTEGER REFERENCES host (id) NOT NULL,
         by_host BOOLEAN NOT NULL,
+        soft BOOLEAN NOT NULL DEFAULT FALSE,
         msg TEXT,
         time TIMESTAMPTZ NOT NULL DEFAULT NOW()
 ) WITHOUT OIDS;

From 5cffbb96871bdaa382e7e9b4e28b36bdf2a50c98 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 59/92] QueryView fragment


---

diff --git a/kojihub/db.py b/kojihub/db.py
index d5e29df..50673d7 100644
--- a/kojihub/db.py
+++ b/kojihub/db.py
@@ -834,6 +834,80 @@ SELECT %(col_str)s
         return results
 
 
+class QueryView:
+    # abstract base class
+
+    # subclasses should provide...
+    tables = []
+    joins = []
+    joinmap = {}
+    fieldmap = {}
+    '''
+    fieldmap looks like:
+
+    {
+        'alias':
+            ['fullname', 'joinkey', 
+
+    }
+    '''
+
+    def __init__(self, fields, clauses, values, opts=None):
+        self.extra_joins = []
+        tables = list(self.tables)  # copy
+        fields = self.get_fields(fields)
+        fields, aliases = zip(*fields.items())
+        clauses = self.get_clauses(clauses)
+        joins = self.get_joins()
+        self.query = QueryProcessor(
+            columns=fields, aliases=aliases,
+            tables=tables, joins=joins,
+            clauses=clauses, values=values,
+            opts=opts)
+
+    def get_fields(self):
+        fields = {}
+        joins = []
+        clauses = []
+
+        x_joins = set()
+        for field in self.fields:
+            f_info = self.fieldmap.get(field)
+            if f_info is None:
+                raise koji.ParameterError(f'Invalid field for query {field}')
+            fullname, joinkey = f_info
+            fullname = fullname or field
+            fields[fullname] = field
+            if joinkey:
+                x_joins.add(joinkey)
+
+    def get_clauses(self):
+        pass
+
+    def get_joins(self):
+        joins = list(self.joins)
+        seen = set()
+        # note we preserve the order that extra joins were added
+        for joinkey in self.extra_joins:
+            if joinkey in seen:
+                continue
+            seen.add(joinkey)
+            joins.append(self.joinmap[joinkey])
+        return joins
+
+    def execute(self):
+        return self.query.execute()
+
+    def executeOne(self, strict=False):
+        return self.query.executeOne(strict=strict)
+
+    def iterate(self):
+        return self.query.iterate()
+
+    def singleValue(self, strict=True):
+        return self.query.singleValue(strict=strict)
+
+
 class BulkInsertProcessor(object):
     def __init__(self, table, data=None, columns=None, strict=True, batch=1000):
         """Do bulk inserts - it has some limitations compared to

From 53d5fc2ac53604f2127bc637fe9c0eafa814dc85 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 60/92] basic QueryView working


---

diff --git a/kojihub/db.py b/kojihub/db.py
index 50673d7..74f03b9 100644
--- a/kojihub/db.py
+++ b/kojihub/db.py
@@ -852,8 +852,9 @@ class QueryView:
     }
     '''
 
-    def __init__(self, fields, clauses, values, opts=None):
+    def __init__(self, clauses=None, fields=None, opts=None):
         self.extra_joins = []
+        self.values = {}
         tables = list(self.tables)  # copy
         fields = self.get_fields(fields)
         fields, aliases = zip(*fields.items())
@@ -862,27 +863,50 @@ class QueryView:
         self.query = QueryProcessor(
             columns=fields, aliases=aliases,
             tables=tables, joins=joins,
-            clauses=clauses, values=values,
+            clauses=clauses, values=self.values,
             opts=opts)
 
-    def get_fields(self):
-        fields = {}
-        joins = []
-        clauses = []
-
-        x_joins = set()
-        for field in self.fields:
-            f_info = self.fieldmap.get(field)
-            if f_info is None:
-                raise koji.ParameterError(f'Invalid field for query {field}')
-            fullname, joinkey = f_info
-            fullname = fullname or field
-            fields[fullname] = field
-            if joinkey:
-                x_joins.add(joinkey)
-
-    def get_clauses(self):
-        pass
+    def get_fields(self, fields):
+        fields = fields or self.fieldmap.keys()  # XXX stable order
+
+        return {self.map_field(f): f for f in fields}
+
+    def map_field(self, field):
+        f_info = self.fieldmap.get(field)
+        if f_info is None:
+            raise koji.ParameterError(f'Invalid field for query {field}')
+        fullname, joinkey = f_info
+        fullname = fullname or field
+        if joinkey:
+            self.extra_joins.append(joinkey)
+        return fullname
+
+    def get_clauses(self, clauses):
+        # for now, just a very simple implementation
+        result = []
+        clauses = clauses or []
+        for n, clause in enumerate(clauses):
+            # TODO checks check checks
+            if len(clause) == 2:
+                # implicit operator
+                field, value = clause
+                if isinstance(value, (list, tuple)):
+                    op = 'IN'
+                else:
+                    op = '='
+            elif len(clause) == 3:
+                field, op, value = clause
+                op = op.upper()
+                if op not in ('IN', '=', '!=', '>', '<', '>=', '<='):
+                    raise koji.ParameterError(f'Invalid operator: {op}')
+            else:
+                raise koji.ParameterError(f'Invalid clause: {clause}')
+            fullname = self.map_field(field)
+            key = f'v_{field}_{n}'
+            self.values[key] = value
+            result.append(f'{fullname} {op} %({key})s')
+
+        return result
 
     def get_joins(self):
         joins = list(self.joins)
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 0283c26..822ad6d 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -4,7 +4,7 @@ import time
 
 import koji
 from . import kojihub
-from .db import QueryProcessor, InsertProcessor, UpdateProcessor, db_lock
+from .db import QueryProcessor, InsertProcessor, UpdateProcessor, QueryView, db_lock
 
 
 logger = logging.getLogger('koji.scheduler')
@@ -125,6 +125,22 @@ def get_host_data(hostID=None):
     return query.execute()
 
 
+class TaskRunsQuery(QueryView):
+
+    tables = ['scheduler_task_runs']
+    fieldmap = {
+        'id': ['scheduler_task_runs.id', None],
+        'task_id': ['scheduler_task_runs.task_id', None],
+        'host_id': ['scheduler_task_runs.host_id', None],
+        'active': ['scheduler_task_runs.active', None],
+        'create_ts': ["date_part('epoch', scheduler_task_runs.create_time)", None],
+    }
+
+
+def get_task_runs2(clauses=None, fields=None):
+    return TaskRunsQuery(clauses, fields).execute()
+
+
 def get_task_runs(taskID=None, hostID=None, active=None):
     taskID = kojihub.convert_value(taskID, cast=int, none_allowed=True)
     hostID = kojihub.convert_value(hostID, cast=int, none_allowed=True)
@@ -548,4 +564,5 @@ class TaskScheduler(object):
 
 class SchedulerExports:
     getTaskRuns = staticmethod(get_task_runs)
+    getTaskRuns2 = staticmethod(get_task_runs2)
     getHostData = staticmethod(get_host_data)

From 308c2ca3e336b33a094393f4a83dac5af1f01c16 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 61/92] ...


---

diff --git a/kojihub/db.py b/kojihub/db.py
index 74f03b9..711619b 100644
--- a/kojihub/db.py
+++ b/kojihub/db.py
@@ -879,6 +879,7 @@ class QueryView:
         fullname = fullname or field
         if joinkey:
             self.extra_joins.append(joinkey)
+            # duplicates removed later
         return fullname
 
     def get_clauses(self, clauses):

From b2cc708fddb03e11cf072334c1db273e9fdef2f8 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 62/92] more QueryView


---

diff --git a/kojihub/db.py b/kojihub/db.py
index 711619b..c812d3a 100644
--- a/kojihub/db.py
+++ b/kojihub/db.py
@@ -842,15 +842,7 @@ class QueryView:
     joins = []
     joinmap = {}
     fieldmap = {}
-    '''
-    fieldmap looks like:
-
-    {
-        'alias':
-            ['fullname', 'joinkey', 
-
-    }
-    '''
+    default_fields = ()
 
     def __init__(self, clauses=None, fields=None, opts=None):
         self.extra_joins = []
@@ -867,7 +859,9 @@ class QueryView:
             opts=opts)
 
     def get_fields(self, fields):
-        fields = fields or self.fieldmap.keys()  # XXX stable order
+        fields = fields or self.default_fields
+        if not fields or fields == '*':
+            fields = sorted(self.fieldmap.keys())
 
         return {self.map_field(f): f for f in fields}
 
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 822ad6d..a2fa0f7 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -128,13 +128,25 @@ def get_host_data(hostID=None):
 class TaskRunsQuery(QueryView):
 
     tables = ['scheduler_task_runs']
+    joinmap = {
+        'task': 'task ON scheduler_task_runs.task_id = task.id',
+        'host': 'host ON scheduler_task_runs.host_id = host.id',
+    }
     fieldmap = {
         'id': ['scheduler_task_runs.id', None],
         'task_id': ['scheduler_task_runs.task_id', None],
+        'method': ['task.method', 'task'],
+        'state': ['task.state', 'task'],
+        'owner': ['task.owner', 'task'],
+        'arch': ['task.arch', 'task'],
+        'channel_id': ['task.channel_id', 'task'],
+        'host_name': ['host.name', 'host'],
+        'host_ready': ['host.ready', 'host'],
         'host_id': ['scheduler_task_runs.host_id', None],
         'active': ['scheduler_task_runs.active', None],
         'create_ts': ["date_part('epoch', scheduler_task_runs.create_time)", None],
     }
+    default_fields = ('id', 'task_id', 'host_id')
 
 
 def get_task_runs2(clauses=None, fields=None):

From 80d4e0f6b7001eaf5351444bc7e36a27036f507d Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 63/92] use QueryView for get_task_runs


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index a2fa0f7..ab7eba8 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -146,45 +146,13 @@ class TaskRunsQuery(QueryView):
         'active': ['scheduler_task_runs.active', None],
         'create_ts': ["date_part('epoch', scheduler_task_runs.create_time)", None],
     }
-    default_fields = ('id', 'task_id', 'host_id')
+    default_fields = ('id', 'task_id', 'host_id', 'active', 'create_ts')
 
 
-def get_task_runs2(clauses=None, fields=None):
+def get_task_runs(clauses=None, fields=None):
     return TaskRunsQuery(clauses, fields).execute()
 
 
-def get_task_runs(taskID=None, hostID=None, active=None):
-    taskID = kojihub.convert_value(taskID, cast=int, none_allowed=True)
-    hostID = kojihub.convert_value(hostID, cast=int, none_allowed=True)
-    active = kojihub.convert_value(active, cast=bool, none_allowed=True)
-
-    fields = (
-        ('scheduler_task_runs.id', 'id'),
-        ('scheduler_task_runs.task_id', 'task_id'),
-        ('scheduler_task_runs.host_id', 'host_id'),
-        # ('host.name', 'host_name'),
-        # ('task.method', 'method'),
-        ('scheduler_task_runs.active', 'active'),
-        ("date_part('epoch', scheduler_task_runs.create_time)", 'create_ts'),
-    )
-    fields, aliases = zip(*fields)
-
-    clauses = []
-    if taskID is not None:
-        clauses.append('task_id = %(taskID)s')
-    if hostID is not None:
-        clauses.append('host_id = %(hostID)s')
-    if active is not None:
-        clauses.append('active = %(active)s')
-
-    query = QueryProcessor(
-        columns=fields, aliases=aliases, tables=['scheduler_task_runs'],
-        # joins=['host ON host_id=host.id', 'task ON task_id=task.id'],
-        clauses=clauses, values=locals())
-
-    return query.execute()
-
-
 class TaskScheduler(object):
 
     def __init__(self):
@@ -427,7 +395,7 @@ class TaskScheduler(object):
             update.execute()
 
     def get_active_runs(self):
-        runs = get_task_runs(active=True)
+        runs = get_task_runs([["active", True]])
         runs_by_task = {}
         for run in runs:
             runs_by_task.setdefault(run['task_id'], [])
@@ -576,5 +544,4 @@ class TaskScheduler(object):
 
 class SchedulerExports:
     getTaskRuns = staticmethod(get_task_runs)
-    getTaskRuns2 = staticmethod(get_task_runs2)
     getHostData = staticmethod(get_host_data)

From ec0fcd870216615f39ef13e3f45cc0662921ae1d Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 64/92] UpsertProcessor


---

diff --git a/kojihub/db.py b/kojihub/db.py
index c812d3a..a7cd1cf 100644
--- a/kojihub/db.py
+++ b/kojihub/db.py
@@ -444,6 +444,42 @@ class InsertProcessor(object):
         return _dml(str(self), self.data)
 
 
+class UpsertProcessor(InsertProcessor):
+    """Build a basic upsert statement
+
+    table - the table to insert into
+    data - a dictionary of data to insert (keys = row names)
+    rawdata - data to insert specified as sql expressions rather than python values
+    keys - the rows that are the unique keys
+    skip_dup - if set to true, do nothing on conflict
+    """
+
+    def __init__(self, table, data=None, rawdata=None, keys=None, skip_dup=False):
+        super(UpsertProcessor, self).__init__(table, data=data, rawdata=rawdata)
+        self.keys = keys
+        self.skip_dup = skip_dup
+        if not keys and not skip_dup:
+            raise ValueError('either keys or skip_dup must be set')
+
+    def __repr__(self):
+        return "<UpsertProcessor: %r>" % vars(self)
+
+    def __str__(self):
+        insert = super(UpsertProcessor, self).__str__()
+        parts = [insert]
+        if self.skip_dup:
+            parts.append(' ON CONFLICT DO NOTHING')
+        else:
+            parts.append(f' ON CONFLICT ({",".join(self.keys)}) DO UPDATE SET ')
+            # filter out conflict keys from data
+            data = {k: self.data[k] for k in self.data if k not in self.keys}
+            rawdata = {k: self.rawdata[k] for k in self.rawdata if k not in self.keys}
+            assigns = [f"{key} = %({key})s" for key in data]
+            assigns.extend([f"{key} = ({rawdata[key]})" for key in self.rawdata])
+            parts.append(', '.join(sorted(assigns)))
+        return ''.join(parts)
+
+
 class UpdateProcessor(object):
     """Build an update statement
 
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index ab7eba8..6646189 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -4,7 +4,7 @@ import time
 
 import koji
 from . import kojihub
-from .db import QueryProcessor, InsertProcessor, UpdateProcessor, QueryView, db_lock
+from .db import QueryProcessor, InsertProcessor, UpsertProcessor, UpdateProcessor, QueryView, db_lock
 
 
 logger = logging.getLogger('koji.scheduler')
@@ -72,36 +72,34 @@ def set_refusal(hostID, taskID, soft=True, by_host=False, msg=''):
     # make very many
 
 
-def get_task_refusals(taskID=None, hostID=None):
-    taskID = kojihub.convert_value(taskID, cast=int, none_allowed=True)
-    hostID = kojihub.convert_value(hostID, cast=int, none_allowed=True)
+class TaskRefusalsQuery(QueryView):
 
-    fields = (
-        ('scheduler_task_refusals.id', 'id'),
-        ('scheduler_task_refusals.task_id', 'task_id'),
-        ('scheduler_task_refusals.host_id', 'host_id'),
-        ('scheduler_task_refusals.by_host', 'by_host'),
-        ('scheduler_task_refusals.soft', 'soft'),
-        ('scheduler_task_refusals.msg', 'msg'),
-        # ('host.name', 'host_name'),
-        ("date_part('epoch', scheduler_task_refusals.time)", 'ts'),
-    )
-    fields, aliases = zip(*fields)
-
-    clauses = []
-    if taskID is not None:
-        clauses.append('task_id = %(taskID)s')
-    if hostID is not None:
-        clauses.append('host_id = %(hostID)s')
+    tables = ['scheduler_task_refusals']
+    joinmap = {
+        'task': 'task ON scheduler_task_refusals.task_id = task.id',
+        'host': 'host ON scheduler_task_refusals.host_id = host.id',
+    }
+    fieldmap = {
+        'id': ['scheduler_task_refusals.id', None],
+        'task_id': ['scheduler_task_refusals.task_id', None],
+        'host_id': ['scheduler_task_refusals.host_id', None],
+        'by_host': ['scheduler_task_refusals.by_host', None],
+        'soft': ['scheduler_task_refusals.soft', None],
+        'msg': ['scheduler_task_refusals.msg', None],
+        'ts': ["date_part('epoch', scheduler_task_refusals.time)", None],
+        'method': ['task.method', 'task'],
+        'state': ['task.state', 'task'],
+        'owner': ['task.owner', 'task'],
+        'arch': ['task.arch', 'task'],
+        'channel_id': ['task.channel_id', 'task'],
+        'host_name': ['host.name', 'host'],
+        'host_ready': ['host.ready', 'host'],
+    }
+    default_fields = ('id', 'task_id', 'host_id', 'by_host', 'soft', 'msg', 'ts')
 
-    query = QueryProcessor(
-        columns=fields, aliases=aliases, tables=['scheduler_task_refusals'],
-        # joins=['host ON host_id=host.id', 'task ON task_id=task.id'],
-        clauses=clauses, values=locals(),
-        opts={'order': '-id'}
-    )
 
-    return query.execute()
+def get_task_refusals(clauses=None, fields=None):
+    return TaskRefusalsQuery(clauses, fields).execute()
 
 
 def get_host_data(hostID=None):
@@ -217,20 +215,13 @@ class TaskScheduler(object):
             ret = True
 
         # save current ts
-        update = UpdateProcessor(
+        upsert = UpsertProcessor(
             'scheduler_sys_data',
-            clauses=['name = %(name)s'],
-            values={'name': 'last_run_ts'},
-            data={'data': json.dumps(now)},
+            data={'name': 'last_run_ts',
+                  'data': json.dumps(now)},
+            keys=['name'],
         )
-        chk = update.execute()
-        if not chk:
-            # hasn't been defined yet
-            insert = InsertProcessor(
-                'scheduler_sys_data',
-                data={'name': 'last_run_ts', 'data': json.dumps(now)},
-            )
-            insert.execute()
+        upsert.execute()
 
         return ret
 
@@ -455,6 +446,9 @@ class TaskScheduler(object):
         self.free_tasks = free_tasks
         self.active_tasks = active_tasks
 
+    def get_refusals(self):
+        pass
+
     def get_hosts(self):
         # get hosts and bin them
         hosts_by_bin = {}
@@ -544,4 +538,5 @@ class TaskScheduler(object):
 
 class SchedulerExports:
     getTaskRuns = staticmethod(get_task_runs)
+    getTaskRefusals = staticmethod(get_task_refusals)
     getHostData = staticmethod(get_host_data)

From 1bc5195e9cc7b238dad821dedddfe862d4b7acf6 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 65/92] honor refusals


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 6646189..33b5568 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -66,10 +66,8 @@ def set_refusal(hostID, taskID, soft=True, by_host=False, msg=''):
         'by_host': kojihub.convert_value(by_host, cast=bool),
         'msg': kojihub.convert_value(msg, cast=str),
     }
-    insert = InsertProcessor('scheduler_task_refusals', data=data)
-    insert.execute()
-    # note: db allows multiple entries here, but in general we shouldn't
-    # make very many
+    upsert = UpsertProcessor('scheduler_task_refusals', data=data, keys=('task_id', 'host_id'))
+    upsert.execute()
 
 
 class TaskRefusalsQuery(QueryView):
@@ -164,6 +162,7 @@ class TaskScheduler(object):
         self.capacity_overcommit = 5
         self.ready_timeout = 180
         self.assign_timeout = 300
+        self.soft_refusal_timeout = 900
         self.host_timeout = 900
         self.run_interval = 60
 
@@ -258,12 +257,15 @@ class TaskScheduler(object):
 
         # figure out which hosts *can* take each task
         # at the moment this is mostly just bin, but in the future it will be more complex
+        refusals = self.get_refusals()
         for task in self.free_tasks:
             task['_hosts'] = []
             min_avail = min(0, task['weight'] - self.capacity_overcommit)
+            h_refused = refusals.get(task['task_id'], {})
             for host in self.hosts_by_bin.get(task['_bin'], []):
                 if (host['ready'] and host['_ntasks'] < self.maxjobs and
-                        host['capacity'] - host['_load'] > min_avail):
+                        host['capacity'] - host['_load'] > min_avail and
+                        host['id'] not in h_refused):
                     task['_hosts'].append(host)
             logger.info(f'Task {task["task_id"]}: {len(task["_hosts"])} options')
             for host in task['_hosts']:
@@ -447,7 +449,31 @@ class TaskScheduler(object):
         self.active_tasks = active_tasks
 
     def get_refusals(self):
-        pass
+        """Get task refusals and clean stale entries"""
+        refusals = {}
+        cutoff_ts = time.time() - self.soft_refusal_timeout
+        to_drop = []
+        for row in get_task_refusals(fields=('id', 'task_id', 'host_id', 'soft', 'ts', 'state')):
+            if ((row['soft'] and row['ts'] < cutoff_ts ) or
+                    koji.TASK_STATES[row['state']] not in ('FREE', 'OPEN', 'ASSIGNED')):
+                to_drop.append(row['id'])
+            else:
+                # index by task and host
+                refusals.setdefault(row['task_id'], {})[row['host_id']] = row
+
+        if to_drop:
+            # drop stale entries
+            delete = DeleteProcessor(
+                'scheduler_task_refusals',
+                clauses=['id IN %(to_drop)s'],
+                values=locals(),
+            )
+            delete.execute()
+
+        return refusals
+
+    def clean_refusals(self):
+        update = UpdateProcessor()
 
     def get_hosts(self):
         # get hosts and bin them
diff --git a/schemas/schema.sql b/schemas/schema.sql
index de97cf6..74addcc 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -1034,7 +1034,8 @@ CREATE TABLE scheduler_task_refusals (
         by_host BOOLEAN NOT NULL,
         soft BOOLEAN NOT NULL DEFAULT FALSE,
         msg TEXT,
-        time TIMESTAMPTZ NOT NULL DEFAULT NOW()
+        time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        UNIQUE (task_id, host_id)
 ) WITHOUT OIDS;
 
 

From 5594e5daa46a7d80a4e98f73e3c4215fafce1ee5 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 66/92] rework host.refuseTask


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index fabbd1c..4b1e0b1 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -14405,15 +14405,22 @@ class HostExports(object):
         tasks = query.execute()
         return tasks
 
-    def refuseTask(self, task_id):
+    def refuseTask(self, task_id, soft=True, msg=''):
+        soft = convert_value(soft, cast=bool)
+        msg = convert_value(msg, cast=str)
         host = Host()
         host.verify()
 
-        # XXX
-
         task = Task(task_id)
-        task.free(newstate=koji.TASK_STATES['REFUSED'])
-        return True
+        if task['host_id'] != host['id']:
+            logger.warning('Host %s refused unrelated task: %s', host['name'], task['id'])
+            return
+        state = koji.TASK_STATES[task['state']]
+        if state not in ('OPEN', 'ASSIGNED'):
+            logger.warning('Host %s refused %s task: %s', host['name'], state, task['id'])
+            return
+        scheduler.set_refusal(host['id'], task['id'], soft=soft, msg=msg, by_host=True)
+        task.free()
 
     def getHostTasks(self):
         host = Host()

From 3d4041315ce6aee49a66efdfc31ae7efc1e861dd Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 67/92] ...


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 4b1e0b1..216d9b0 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -14412,14 +14412,12 @@ class HostExports(object):
         host.verify()
 
         task = Task(task_id)
-        if task['host_id'] != host['id']:
-            logger.warning('Host %s refused unrelated task: %s', host['name'], task['id'])
+        tinfo = task.getInfo(strict=True)
+        if tinfo['host_id'] != host.id:
+            logger.warning('Host %s refused unrelated task: %s', host.id, tinfo['id'])
             return
-        state = koji.TASK_STATES[task['state']]
-        if state not in ('OPEN', 'ASSIGNED'):
-            logger.warning('Host %s refused %s task: %s', host['name'], state, task['id'])
-            return
-        scheduler.set_refusal(host['id'], task['id'], soft=soft, msg=msg, by_host=True)
+        scheduler.set_refusal(host.id, tinfo['id'], soft=soft, msg=msg, by_host=True)
+        # also free the task
         task.free()
 
     def getHostTasks(self):

From f2f91ec46aa59c1036299f0adf31253e387d0642 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 68/92] call host.refuseTask when host check fails


---

diff --git a/koji/daemon.py b/koji/daemon.py
index fe71203..1d9b313 100644
--- a/koji/daemon.py
+++ b/koji/daemon.py
@@ -1429,6 +1429,8 @@ class TaskManager(object):
             if not valid_host:
                 self.logger.info(
                     'Skipping task %s (%s) due to host check', task['id'], task['method'])
+                if task['state'] == koji.TASK_STATES['ASSIGNED']:
+                    self.session.host.refuseTask(task_id, soft=False, msg='failed host check')
                 return False
         data = self.session.host.openTask(task['id'])
         if data is None:

From cbab23961200bb46ac395f0ebddbb361af0bfbd6 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 69/92] query for scheduler logs


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 33b5568..1275d3b 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -25,6 +25,34 @@ def log_both(msg, task_id=None, host_id=None, level=logging.INFO):
     log_db(msg, task_id, host_id)
 
 
+class LogMessagesQuery(QueryView):
+
+    tables = ['scheduler_log_messages']
+    joinmap = {
+        'task': 'task ON scheduler_log_messages.task_id = task.id',
+        'host': 'host ON scheduler_log_messages.host_id = host.id',
+    }
+    fieldmap = {
+        'id': ['scheduler_log_messages.id', None],
+        'task_id': ['scheduler_log_messages.task_id', None],
+        'host_id': ['scheduler_log_messages.host_id', None],
+        'msg_ts': ["date_part('epoch', scheduler_log_messages.msg_time)", None],
+        'msg': ['scheduler_log_messages.msg', None],
+        'method': ['task.method', 'task'],
+        'state': ['task.state', 'task'],
+        'owner': ['task.owner', 'task'],
+        'arch': ['task.arch', 'task'],
+        'channel_id': ['task.channel_id', 'task'],
+        'host_name': ['host.name', 'host'],
+        'host_ready': ['host.ready', 'host'],
+    }
+    default_fields = ('id', 'task_id', 'host_id', 'msg', 'msg_ts')
+
+
+def get_log_messages(clauses=None, fields=None):
+    return LogMessagesQuery(clauses, fields).execute()
+
+
 def intlist(value):
     """Cast value to a list of ints"""
     if isinstance(value, (list, tuple)):
@@ -68,6 +96,7 @@ def set_refusal(hostID, taskID, soft=True, by_host=False, msg=''):
     }
     upsert = UpsertProcessor('scheduler_task_refusals', data=data, keys=('task_id', 'host_id'))
     upsert.execute()
+    log_both('Host refused task', task_id=taskID, host_id=hostID)
 
 
 class TaskRefusalsQuery(QueryView):
@@ -472,9 +501,6 @@ class TaskScheduler(object):
 
         return refusals
 
-    def clean_refusals(self):
-        update = UpdateProcessor()
-
     def get_hosts(self):
         # get hosts and bin them
         hosts_by_bin = {}
@@ -566,3 +592,4 @@ class SchedulerExports:
     getTaskRuns = staticmethod(get_task_runs)
     getTaskRefusals = staticmethod(get_task_refusals)
     getHostData = staticmethod(get_host_data)
+    getLogMessages = staticmethod(get_log_messages)

From 8ad6993a873cee9807ec232c2bdd3a7ccf4e6acf Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 70/92] use host.update_ts in kojiweb/hostinfo


---

diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py
index 35ed162..0e0b3ae 100644
--- a/www/kojiweb/index.py
+++ b/www/kojiweb/index.py
@@ -1780,7 +1780,12 @@ def hostinfo(environ, hostID=None, userID=None):
     values['host'] = host
     values['channels'] = channels
     values['buildroots'] = buildroots
-    values['lastUpdate'] = server.getLastHostUpdate(host['id'], ts=True)
+    if 'update_ts' not in host:
+        # be nice with older hub
+        # TODO remove this compat workaround after a release
+        values['lastUpdate'] = server.getLastHostUpdate(host['id'], ts=True)
+    else:
+        values['lastUpdate'] = koji.formatTimeLong(host['update_ts'])
     if environ['koji.currentUser']:
         values['perms'] = server.getUserPerms(environ['koji.currentUser']['id'])
     else:

From 9c68bb22c58a7ae78e4419d46f996a2af2f1dffb Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 71/92] basic scheduler log cli


---

diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py
index bc14d40..2dc2cc6 100644
--- a/cli/koji_cli/commands.py
+++ b/cli/koji_cli/commands.py
@@ -7955,3 +7955,53 @@ def anon_handle_scheduler_info(goptions, session, args):
             print(host_data[0]['data'])
         else:
             print('-')
+
+
+def handle_scheduler_logs(goptions, session, args):
+    "[monitor] Query scheduler logs"
+    usage = "usage: %prog scheduler-logs <options>"
+    parser = OptionParser(usage=get_usage_str(usage))
+    parser.add_option("--task", type="int", action="store",
+                      help="Filter by task ID")
+    parser.add_option("--host", type="str", action="store",
+                      help="Filter by host (name/ID)")
+    parser.add_option("--from", type="float", action="store", dest="from_ts",
+                      help="Logs from given timestamp")
+    parser.add_option("--to", type="float", action="store", dest="to_ts",
+                      help="Logs until given timestamp (included)")
+    (options, args) = parser.parse_args(args)
+    if len(args) != 0:
+        parser.error("There are no arguments for this command")
+
+    kwargs = {}
+    clauses = []
+    if options.task:
+        clauses.append(['task_id', options.task])
+    if options.host:
+        try:
+            host_id = int(options.host)
+        except ValueError:
+            host_id = session.getHost(options.host)['id']
+        clauses.append(['host_id', host_id])
+    if options.from_ts:
+        clauses.append(['msg_ts', '>=', options.from_ts])
+    if options.to_ts:
+        clauses.append(['msg_ts', '<', options.to_ts])
+
+    logs = session.scheduler.getLogMessages(clauses, fields=('task_id', 'host_id', 'host_name', 'msg_ts', 'msg'))
+    for log in logs:
+        log['time'] = time.asctime(time.localtime(log['msg_ts']))
+
+    mask = ("%(task_id)s\t%(host_name)s\t%(time)s\t%(msg)s")
+    if not goptions.quiet:
+        h = mask % {
+            'task_id': 'Task',
+            'host_name': 'Host',
+            'time': 'Time',
+            'msg': 'Message',
+        }
+        print(h)
+        print('-' * len(h))
+
+    for log in logs:
+        print(mask % log)

From 43718d90a8335cf9d9376d176d00c0579f8ea722 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 72/92] outer joins for log query


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 1275d3b..eb91f82 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -29,8 +29,9 @@ class LogMessagesQuery(QueryView):
 
     tables = ['scheduler_log_messages']
     joinmap = {
-        'task': 'task ON scheduler_log_messages.task_id = task.id',
-        'host': 'host ON scheduler_log_messages.host_id = host.id',
+        # outer joins because these fields can be null
+        'task': 'LEFT JOIN task ON scheduler_log_messages.task_id = task.id',
+        'host': 'LEFT JOIN host ON scheduler_log_messages.host_id = host.id',
     }
     fieldmap = {
         'id': ['scheduler_log_messages.id', None],

From a668e93be75ac05e97ae425039cfe92e259f949d Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 73/92] drop unused task states


---

diff --git a/koji/__init__.py b/koji/__init__.py
index 6f1aa83..4d0e7b2 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -194,8 +194,6 @@ TASK_STATES = Enum((
     'CANCELED',
     'ASSIGNED',
     'FAILED',
-    'SCHEDULED',
-    'REFUSED',
 ))
 
 BUILD_STATES = Enum((

From f3bdc8b40d89b499b7dc7f960d904bc4aae16ce8 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 74/92] fix import


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index eb91f82..798f6b0 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -4,7 +4,8 @@ import time
 
 import koji
 from . import kojihub
-from .db import QueryProcessor, InsertProcessor, UpsertProcessor, UpdateProcessor, QueryView, db_lock
+from .db import QueryProcessor, InsertProcessor, UpsertProcessor, UpdateProcessor, \
+    DeleteProcessor, QueryView, db_lock
 
 
 logger = logging.getLogger('koji.scheduler')

From 234987412c9e7c26f008c85d4404af30613be583 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 75/92] fix schema


---

diff --git a/schemas/schema.sql b/schemas/schema.sql
index 74addcc..7bed75f 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -1000,7 +1000,7 @@ CREATE INDEX scheduler_task_runs_create_time ON scheduler_task_runs(create_time)
 CREATE TABLE scheduler_queue (
         task_id INTEGER REFERENCES task (id) NOT NULL PRIMARY KEY,
         groomed BOOLEAN NOT NULL DEFAULT FALSE,
-        run_id INTEGER REFERENCES scheduler_task_run(id)
+        run_id INTEGER REFERENCES scheduler_task_runs(id)
 ) WITHOUT OIDS;
 
 

From cb5e40466061817f1346f0514ba72082e20fd044 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 76/92] fix scheduler-info


---

diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py
index 2dc2cc6..441fff9 100644
--- a/cli/koji_cli/commands.py
+++ b/cli/koji_cli/commands.py
@@ -7901,6 +7901,12 @@ def anon_handle_repoinfo(goptions, session, args):
                 warn("--buildroots option is available with hub 1.33 or newer")
 
 
+def _format_ts(ts):
+    if ts:
+        return time.strftime("%y-%m-%d %H:%M:%S", time.localtime(ts))
+    else:
+        return ''
+
 def anon_handle_scheduler_info(goptions, session, args):
     """[monitor] Show information about scheduling"""
     usage = "usage: %prog schedulerinfo [options]"
@@ -7908,8 +7914,8 @@ def anon_handle_scheduler_info(goptions, session, args):
     parser.add_option("-t", "--task", action="store", type=int, default=None,
                       help="Limit data to given task id")
     parser.add_option("--host", action="store", default=None,
-                      help="Limit data to given builder (name/id)")
-    parser.add_option("--state", action="store", type='str', default=None,
+                      help="Limit data to given builder id")
+    parser.add_option("--state", action="store", type='choice', default=None,
                       choices=[x for x in koji.TASK_STATES.keys()],
                       help="Limit data to task state")
     (options, args) = parser.parse_args(args)
@@ -7925,27 +7931,36 @@ def anon_handle_scheduler_info(goptions, session, args):
         except ValueError:
             host_id = session.getHost(options.host, strict=True)['id']
 
-    if options.state:
-        state = koji.TASK_STATES[options.state]
-    else:
-        state = None
-
     # get the data
-    runs = session.scheduler.getTaskRuns(taskID=options.task, hostID=host_id, state=state)
-    mask = '%(task_id)s\t%(host_id)s\t%(state)s\t%(create_time)s\t%(start_time)s\t%(end_time)s'
+    clauses = []
+    if options.task:
+        clauses.append(('task_id', options.task))
+    if options.host:
+        clauses.append(('host_id', options.host))
+    if options.state:
+        clauses.append(('state', koji.TASK_STATES[options.state]))
+
+    runs = session.scheduler.getTaskRuns(
+        clauses=clauses,
+        fields=('task_id', 'host_name', 'state', 'create_ts', 'start_ts', 'completion_ts')
+    )
+    mask = '%(task_id)-9s %(host_name)-20s %(state)-7s ' \
+           '%(create_ts)-17s %(start_ts)-17s %(completion_ts)-17s'
     if not goptions.quiet:
         header = mask % {
             'task_id': 'Task',
             'host_name': 'Host',
             'state': 'State',
-            'create_time': 'Created',
-            'start_time': 'Started',
-            'end_time': 'Ended'
+            'create_ts': 'Created',
+            'start_ts': 'Started',
+            'completion_ts': 'Ended',
         }
         print(header)
         print('-' * len(header))
     for run in runs:
-        run['state'] = koji.TASK_STATES[runs['state']]
+        run['state'] = koji.TASK_STATES[run['state']]
+        for ts in ('create_ts', 'start_ts', 'completion_ts'):
+            run[ts] = _format_ts(run[ts])
         print(mask % run)
 
     if host_id:
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 798f6b0..08a572b 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -172,6 +172,8 @@ class TaskRunsQuery(QueryView):
         'host_id': ['scheduler_task_runs.host_id', None],
         'active': ['scheduler_task_runs.active', None],
         'create_ts': ["date_part('epoch', scheduler_task_runs.create_time)", None],
+        'start_ts': ["date_part('epoch', task.start_time)", 'task'],
+        'completion_ts': ["date_part('epoch', task.completion_time)", 'task'],
     }
     default_fields = ('id', 'task_id', 'host_id', 'active', 'create_ts')
 

From 4ffba4df4e0962beecfe493427d2211f70c65f4d Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 77/92] use UpsertProcessor


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 216d9b0..1a7ed04 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -83,6 +83,7 @@ from .db import (  # noqa: F401
     QueryProcessor,
     Savepoint,
     UpdateProcessor,
+    UpsertProcessor,
     _applyQueryOpts,
     _dml,
     _fetchSingle,
@@ -14370,19 +14371,12 @@ class HostExports(object):
         """
         host = Host()
         host.verify()
-        clauses = ['host_id = %(host_id)i']
-        values = {'host_id': host.id}
-        table = 'scheduler_host_data'
-        query = QueryProcessor(tables=[table], clauses=clauses, values=values,
-                               opts={'countOnly': True})
-        if query.singleValue() > 0:
-            update = UpdateProcessor(table=table, data={'data': hostdata},
-                                     clauses=clauses, values=values)
-            update.execute()
-        else:
-            insert = InsertProcessor(table=table, data={'data': hostdata},
-                                     clauses=clauses, values=values)
-            insert.execute()
+        upsert = UpsertProcessor(
+            table='scheduler_host_data',
+            keys=['host_id'],
+            data={'host_id': host.id, 'data': hostdata},
+        )
+        upsert.execute()
 
     def getTasks(self):
         host = Host()

From ca352e29530296e322849c28e15bbc99bef4dbeb Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 78/92] use Task interface for correct state hooks


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 08a572b..89d89e0 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -583,13 +583,8 @@ class TaskScheduler(object):
         insert.execute()
 
         # mark the task assigned
-        update = UpdateProcessor(
-            'task',
-            data={'host_id': host['id'], 'state': koji.TASK_STATES['ASSIGNED']},
-            clauses=['id=%(task_id)s', 'state=%(free)s'],
-            values={'task_id': task['task_id'], 'free': koji.TASK_STATES['FREE']},
-        )
-        update.execute()
+        task = kojihub.Task(task['task_id'])
+        task.assign(host['id'])
 
 
 class SchedulerExports:

From 36f18c721dd110d35150f4550e6f3df78ef8e667 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 79/92] allow admins to trigger scheduler run with an api call


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 89d89e0..a207a5a 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -3,6 +3,7 @@ import logging
 import time
 
 import koji
+from koji.context import context
 from . import kojihub
 from .db import QueryProcessor, InsertProcessor, UpsertProcessor, UpdateProcessor, \
     DeleteProcessor, QueryView, db_lock
@@ -199,12 +200,12 @@ class TaskScheduler(object):
         self.host_timeout = 900
         self.run_interval = 60
 
-    def run(self):
-        if not db_lock('scheduler', wait=False):
+    def run(self, force=False):
+        if not db_lock('scheduler', wait=force):
             # already running elsewhere
             return False
 
-        if not self.check_ts():
+        if not force and not self.check_ts():
             # already ran too recently
             return False
 
@@ -592,3 +593,14 @@ class SchedulerExports:
     getTaskRefusals = staticmethod(get_task_refusals)
     getHostData = staticmethod(get_host_data)
     getLogMessages = staticmethod(get_log_messages)
+
+    def doRun(self, force=False):
+        """Run the scheduler
+
+        This is a debug tool and should not normally be needed.
+        Scheduler runs are regularly triggered by builder checkins
+        """
+
+        force = kojihub.convert_value(force, cast=bool)
+        context.session.assertPerm('admin')
+        return TaskScheduler().run(force=force)

From ffd337d0ed2a2b0eeb1c46001e4ffd0a1fd69d00 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 80/92] drop some test code


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 1a7ed04..60cabf0 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -10047,15 +10047,6 @@ def _delete_event_id():
 class RootExports(object):
     '''Contains functions that are made available via XMLRPC'''
 
-    def TEST(self, fail=False):
-        scheduler.TaskScheduler().run()
-        if fail:
-            raise Exception('DEBUG')
-
-    def TEST2(self, hostID=1):
-        return Host(hostID).getLoadData()
-        # return scheduler.get_tasks_for_host(hostID)
-
     def restartHosts(self, priority=5, options=None):
         """Spawns restartHosts task
 

From 67f2c00c4703c11e54d1680319a78f933e1e2945 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 81/92] unify getTasks and getLoadData code


---

diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py
index 60cabf0..37e5be8 100644
--- a/kojihub/kojihub.py
+++ b/kojihub/kojihub.py
@@ -14268,16 +14268,16 @@ class Host(object):
     def getLoadData(self):
         """Get load balancing data
 
-        This data is relatively small and the necessary load analysis is
-        relatively complex, so we let the host machines crunch it."""
+        This call is here for backwards compatibility.
+        Originally, it returned broad information about all hosts and tasks so that individual
+        hosts could make informed decisions about which task to take.
+
+        Now it presents only data for the calling host and the tasks that have been assigned to
+        it"""
+
         host = get_host(self.id)
         host['channels'] = [c['id'] for c in list_channels(hostID=self.id)]
-        tasks = scheduler.get_tasks_for_host(hostID=self.id)
-        if not tasks:
-            # try running scheduler
-            if scheduler.TaskScheduler().run():
-                # check again
-                tasks = scheduler.get_tasks_for_host(hostID=self.id)
+        tasks = scheduler.get_tasks_for_host(hostID=self.id, retry=True)
         return [[host], tasks]
 
     def isEnabled(self):
@@ -14372,23 +14372,7 @@ class HostExports(object):
     def getTasks(self):
         host = Host()
         host.verify()
-
-        query = QueryProcessor(
-            tables=['scheduler_task_runs'],
-            clauses=[
-                'host_id = %(host_id)s',
-                'state in %(states)s'
-            ],
-            values={
-                'host_id': host.id,
-                'states': [
-                    koji.TASK_STATES['SCHEDULED'],
-                    koji.TASK_STATES['ASSIGNED'],
-                ],
-            }
-        )
-        tasks = query.execute()
-        return tasks
+        return scheduler.get_tasks_for_host(hostID=host.id, retry=True)
 
     def refuseTask(self, task_id, soft=True, msg=''):
         soft = convert_value(soft, cast=bool)
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index a207a5a..db535cb 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -64,7 +64,7 @@ def intlist(value):
         return [int(value)]
 
 
-def get_tasks_for_host(hostID):
+def get_tasks_for_host(hostID, retry=True):
     """Get the tasks assigned to a given host"""
     hostID = kojihub.convert_value(hostID, cast=int, none_allowed=True)
 
@@ -86,7 +86,14 @@ def get_tasks_for_host(hostID):
         values={'hostID': hostID, 'assigned': koji.TASK_STATES['ASSIGNED']},
     )
 
-    return query.execute()
+    tasks = query.execute()
+
+    if not tasks and retry:
+        # run scheduler and try again
+        TaskScheduler().run()
+        tasks = query.execute()
+
+    return tasks
 
 
 def set_refusal(hostID, taskID, soft=True, by_host=False, msg=''):

From 1f472b356341fb0406cb6a2f61400e7fa716abab Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 82/92] drop unused code


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index db535cb..33fd7d2 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -56,14 +56,6 @@ def get_log_messages(clauses=None, fields=None):
     return LogMessagesQuery(clauses, fields).execute()
 
 
-def intlist(value):
-    """Cast value to a list of ints"""
-    if isinstance(value, (list, tuple)):
-        return [int(n) for n in value]
-    else:
-        return [int(value)]
-
-
 def get_tasks_for_host(hostID, retry=True):
     """Get the tasks assigned to a given host"""
     hostID = kojihub.convert_value(hostID, cast=int, none_allowed=True)
diff --git a/schemas/schema.sql b/schemas/schema.sql
index 7bed75f..e58ad50 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -997,13 +997,6 @@ CREATE INDEX scheduler_task_runs_host ON scheduler_task_runs(host_id);
 CREATE INDEX scheduler_task_runs_create_time ON scheduler_task_runs(create_time);
 
 
-CREATE TABLE scheduler_queue (
-        task_id INTEGER REFERENCES task (id) NOT NULL PRIMARY KEY,
-        groomed BOOLEAN NOT NULL DEFAULT FALSE,
-        run_id INTEGER REFERENCES scheduler_task_runs(id)
-) WITHOUT OIDS;
-
-
 CREATE TABLE scheduler_host_data (
         host_id INTEGER REFERENCES host (id) PRIMARY KEY,
         data JSONB
@@ -1016,17 +1009,6 @@ CREATE TABLE scheduler_sys_data (
 ) WITHOUT OIDS;
 
 
-CREATE TABLE scheduler_map (
-        id SERIAL NOT NULL PRIMARY KEY,
-        task_id INTEGER REFERENCES task (id) NOT NULL,
-        host_id INTEGER REFERENCES host (id) NOT NULL,
-        runnable BOOLEAN NOT NULL,
-        refused BOOLEAN NOT NULL,
-        msg TEXT,
-        check_time TIMESTAMPTZ NOT NULL DEFAULT NOW()
-) WITHOUT OIDS;
-
-
 CREATE TABLE scheduler_task_refusals (
         id SERIAL NOT NULL PRIMARY KEY,
         task_id INTEGER REFERENCES task (id) NOT NULL,

From ee5f3900aabdfa1612d43ef4741bc1eaef9988a0 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 83/92] fix unit tests


get_ready_hosts function was dropped

---

diff --git a/tests/test_hub/test_get_ready_hosts.py b/tests/test_hub/test_get_ready_hosts.py
deleted file mode 100644
index 7ae8843..0000000
--- a/tests/test_hub/test_get_ready_hosts.py
+++ /dev/null
@@ -1,64 +0,0 @@
-import kojihub
-import mock
-import unittest
-
-QP = kojihub.QueryProcessor
-
-
-class TestGetReadyHosts(unittest.TestCase):
-
-    def setUp(self):
-        self.maxDiff = None
-        self.QueryProcessor = mock.patch('kojihub.kojihub.QueryProcessor',
-                                         side_effect=self.getQuery).start()
-        self.queries = []
-        self.query_execute = mock.MagicMock()
-
-    def tearDown(self):
-        mock.patch.stopall()
-
-    def getQuery(self, *args, **kwargs):
-        query = QP(*args, **kwargs)
-        query.execute = self.query_execute
-        self.queries.append(query)
-        return query
-
-    def test_valid(self):
-        hosts = [{'host.id': 1, 'name': 'hostname', 'arches': 'arch123', 'task_load': 0,
-                  'capacity': 3},
-                 {'host.id': 2, 'name': 'hostname-2', 'arches': 'arch123', 'task_load': 0,
-                  'capacity': 3}]
-        expected_res = [{'host.id': 1, 'name': 'hostname', 'arches': 'arch123', 'task_load': 0,
-                         'capacity': 3, 'channels': [1]},
-                        {'host.id': 2, 'name': 'hostname-2', 'arches': 'arch123', 'task_load': 0,
-                         'capacity': 3, 'channels': [2, 3]}
-                        ]
-        self.query_execute.side_effect = [hosts, [{'channel_id': 1}],
-                                          [{'channel_id': 2}, {'channel_id': 3}]]
-        result = kojihub.get_ready_hosts()
-        self.assertEqual(result, expected_res)
-        self.assertEqual(len(self.queries), 3)
-
-        query = self.queries[0]
-        self.assertEqual(query.tables, ['host'])
-        self.assertEqual(query.joins, ['sessions USING (user_id)',
-                                       'host_config ON host.id = host_config.host_id'])
-        self.assertEqual(query.clauses, ['active IS TRUE', 'enabled IS TRUE', 'expired IS FALSE',
-                                         'master IS NULL', 'ready IS TRUE',
-                                         "sessions.update_time > NOW() - '5 minutes'::interval"])
-        self.assertEqual(query.values, {})
-        self.assertEqual(query.columns, ['arches', 'capacity', 'host.id', 'name', 'task_load'])
-
-        query = self.queries[1]
-        self.assertEqual(query.tables, ['host_channels'])
-        self.assertEqual(query.joins, ['channels ON host_channels.channel_id = channels.id'])
-        self.assertEqual(query.clauses, ['active IS TRUE', 'enabled IS TRUE', 'host_id=%(id)s'])
-        self.assertEqual(query.values, hosts[0])
-        self.assertEqual(query.columns, ['channel_id'])
-
-        query = self.queries[2]
-        self.assertEqual(query.tables, ['host_channels'])
-        self.assertEqual(query.joins, ['channels ON host_channels.channel_id = channels.id'])
-        self.assertEqual(query.clauses, ['active IS TRUE', 'enabled IS TRUE', 'host_id=%(id)s'])
-        self.assertEqual(query.values, hosts[1])
-        self.assertEqual(query.columns, ['channel_id'])

From 80d37400850a5a3853835e76df1509ba167c4596 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 84/92] unit test


---

diff --git a/tests/test_cli/data/list-commands.txt b/tests/test_cli/data/list-commands.txt
index c434b68..ed8a656 100644
--- a/tests/test_cli/data/list-commands.txt
+++ b/tests/test_cli/data/list-commands.txt
@@ -136,6 +136,8 @@ monitor commands:
         edit-notification         Edit user's notification
         list-notifications        List user's notifications and blocks
         remove-notification       Remove user's notifications
+        scheduler-info            Show information about scheduling
+        scheduler-logs            Query scheduler logs
         unblock-notification      Unblock user's notification
         wait-repo                 Wait for a repo to be regenerated
         watch-logs                Watch logs in realtime

From 702391100b524ccdb36d90cd661ce1d1623c18c7 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 85/92] schema upgrade


---

diff --git a/docs/schema-upgrade-1.33-1.34.sql b/docs/schema-upgrade-1.33-1.34.sql
new file mode 100644
index 0000000..ad29d0b
--- /dev/null
+++ b/docs/schema-upgrade-1.33-1.34.sql
@@ -0,0 +1,49 @@
+-- upgrade script to migrate the Koji database schema
+-- from version 1.33 to 1.34
+
+BEGIN;
+    -- scheduler tables
+    CREATE TABLE scheduler_task_runs (
+            id SERIAL NOT NULL PRIMARY KEY,
+            task_id INTEGER REFERENCES task (id) NOT NULL,
+            host_id INTEGER REFERENCES host (id) NOT NULL,
+            active BOOLEAN NOT NULL DEFAULT TRUE,
+            create_time TIMESTAMPTZ NOT NULL DEFAULT NOW()
+    ) WITHOUT OIDS;
+    CREATE INDEX scheduler_task_runs_task ON scheduler_task_runs(task_id);
+    CREATE INDEX scheduler_task_runs_host ON scheduler_task_runs(host_id);
+    CREATE INDEX scheduler_task_runs_create_time ON scheduler_task_runs(create_time);
+
+
+    CREATE TABLE scheduler_host_data (
+            host_id INTEGER REFERENCES host (id) PRIMARY KEY,
+            data JSONB
+    ) WITHOUT OIDS;
+
+
+    CREATE TABLE scheduler_sys_data (
+            name TEXT NOT NULL PRIMARY KEY,
+            data JSONB
+    ) WITHOUT OIDS;
+
+
+    CREATE TABLE scheduler_task_refusals (
+            id SERIAL NOT NULL PRIMARY KEY,
+            task_id INTEGER REFERENCES task (id) NOT NULL,
+            host_id INTEGER REFERENCES host (id) NOT NULL,
+            by_host BOOLEAN NOT NULL,
+            soft BOOLEAN NOT NULL DEFAULT FALSE,
+            msg TEXT,
+            time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+            UNIQUE (task_id, host_id)
+    ) WITHOUT OIDS;
+
+
+    CREATE TABLE scheduler_log_messages (
+            id SERIAL NOT NULL PRIMARY KEY,
+            task_id INTEGER REFERENCES task (id),
+            host_id INTEGER REFERENCES host (id),
+            msg_time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+            msg TEXT NOT NULL
+    ) WITHOUT OIDS;
+COMMIT;

From bc8cb46eceaf550e50304088d38f5144bafd1b50 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 86/92] default values


---

diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 33fd7d2..e8fb9b0 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -185,10 +185,10 @@ def get_task_runs(clauses=None, fields=None):
 class TaskScheduler(object):
 
     def __init__(self):
-        self.hosts_by_bin = None
-        self.hosts = None
-        self.active_tasks = None
-        self.free_tasks = None
+        self.hosts_by_bin = {}
+        self.hosts = {}
+        self.active_tasks = []
+        self.free_tasks = []
 
         # TODO these things need proper config
         self.maxjobs = 15  # XXX

From 483e5715976ebf4105174261a0a7bda89c841c63 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 87/92] remove typo


---

diff --git a/util/koji-sweep-db b/util/koji-sweep-db
index 373bb8e..8c57515 100755
--- a/util/koji-sweep-db
+++ b/util/koji-sweep-db
@@ -156,7 +156,7 @@ def clean_scheduler_logs(cursor, vacuum, test, age):
         rows = query.execute()
         print(f"Deleting {rows} scheduler log messages")
     if not test:
-        delete = DeleteProcessor(table="scheduler_log_messages", clauses=clauses)\
+        delete = DeleteProcessor(table="scheduler_log_messages", clauses=[clauses])
         delete.execute()
         if vacuum:
             cursor.execute("VACUUM ANALYZE scheduler_log_messages")

From 0b56c8e751698724a215729141b4558dbdd48331 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 88/92] unify clauses handling


---

diff --git a/util/koji-sweep-db b/util/koji-sweep-db
index 8c57515..11a759d 100755
--- a/util/koji-sweep-db
+++ b/util/koji-sweep-db
@@ -11,17 +11,17 @@ from kojihub.db import DeleteProcessor, QueryProcessor, BulkInsertProcessor
 
 
 def clean_sessions(cursor, vacuum, test, age, absolute):
-    clauses = f"(update_time < NOW() - '{age:d} days'::interval)"
+    clause = f"(update_time < NOW() - '{age:d} days'::interval)"
     if absolute is not None:
-        clauses += f"OR (start_time < NOW() - '{absolute:d} days'::interval)"
+        clause += f"OR (start_time < NOW() - '{absolute:d} days'::interval)"
 
     if options.verbose:
-        query = QueryProcessor(tables=['sessions'], clauses=[clauses], opts={'countOnly': True})
+        query = QueryProcessor(tables=['sessions'], clauses=[clause], opts={'countOnly': True})
         rows = query.execute()
         print(f"Deleting {rows} sessions")
 
     if not test:
-        delete = DeleteProcessor(table='sessions', clauses=[clauses])
+        delete = DeleteProcessor(table='sessions', clauses=[clause])
         delete.execute()
         if vacuum:
             cursor.execute("VACUUM ANALYZE sessions")
@@ -148,7 +148,7 @@ def clean_buildroots(cursor, vacuum, test):
 
 
 def clean_scheduler_logs(cursor, vacuum, test, age):
-    clauses = f"(msg_time < NOW() - '{age:d} days'::interval)"
+    clauses = [f"(msg_time < NOW() - '{age:d} days'::interval)"]
     if options.verbose:
         query = QueryProcessor(tables=["scheduler_log_messages"],
                                clauses=clauses,
@@ -156,7 +156,7 @@ def clean_scheduler_logs(cursor, vacuum, test, age):
         rows = query.execute()
         print(f"Deleting {rows} scheduler log messages")
     if not test:
-        delete = DeleteProcessor(table="scheduler_log_messages", clauses=[clauses])
+        delete = DeleteProcessor(table="scheduler_log_messages", clauses=clauses)
         delete.execute()
         if vacuum:
             cursor.execute("VACUUM ANALYZE scheduler_log_messages")

From 559c41eca87a5e7d59c73dd39280e2ffecf0b27f Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 89/92] missing lock in db


---

diff --git a/docs/schema-upgrade-1.33-1.34.sql b/docs/schema-upgrade-1.33-1.34.sql
index ad29d0b..ac02893 100644
--- a/docs/schema-upgrade-1.33-1.34.sql
+++ b/docs/schema-upgrade-1.33-1.34.sql
@@ -46,4 +46,6 @@ BEGIN;
             msg_time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
             msg TEXT NOT NULL
     ) WITHOUT OIDS;
+
+    INSERT INTO locks(name) VALUES('scheduler');
 COMMIT;
diff --git a/schemas/schema.sql b/schemas/schema.sql
index e58ad50..a007b2c 100644
--- a/schemas/schema.sql
+++ b/schemas/schema.sql
@@ -1035,5 +1035,6 @@ CREATE TABLE locks (
         name TEXT NOT NULL PRIMARY KEY
 ) WITHOUT OIDS;
 INSERT INTO locks(name) VALUES('protonmsg-plugin');
+INSERT INTO locks(name) VALUES('scheduler');
 
 COMMIT WORK;

From eee19817a6f7aed54bcf61557c8be3e343334869 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:47 +0000
Subject: [PATCH 90/92] wrong variable


---

diff --git a/koji/daemon.py b/koji/daemon.py
index 1d9b313..a7a9561 100644
--- a/koji/daemon.py
+++ b/koji/daemon.py
@@ -1430,7 +1430,7 @@ class TaskManager(object):
                 self.logger.info(
                     'Skipping task %s (%s) due to host check', task['id'], task['method'])
                 if task['state'] == koji.TASK_STATES['ASSIGNED']:
-                    self.session.host.refuseTask(task_id, soft=False, msg='failed host check')
+                    self.session.host.refuseTask(task['id'], soft=False, msg='failed host check')
                 return False
         data = self.session.host.openTask(task['id'])
         if data is None:

From f68a5275a627ff5cfcff464a01b5752901d049be Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:48 +0000
Subject: [PATCH 91/92] flake8 fixes


---

diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py
index 441fff9..9f64249 100644
--- a/cli/koji_cli/commands.py
+++ b/cli/koji_cli/commands.py
@@ -7907,6 +7907,7 @@ def _format_ts(ts):
     else:
         return ''
 
+
 def anon_handle_scheduler_info(goptions, session, args):
     """[monitor] Show information about scheduling"""
     usage = "usage: %prog schedulerinfo [options]"
@@ -7988,7 +7989,6 @@ def handle_scheduler_logs(goptions, session, args):
     if len(args) != 0:
         parser.error("There are no arguments for this command")
 
-    kwargs = {}
     clauses = []
     if options.task:
         clauses.append(['task_id', options.task])
@@ -8003,7 +8003,8 @@ def handle_scheduler_logs(goptions, session, args):
     if options.to_ts:
         clauses.append(['msg_ts', '<', options.to_ts])
 
-    logs = session.scheduler.getLogMessages(clauses, fields=('task_id', 'host_id', 'host_name', 'msg_ts', 'msg'))
+    logs = session.scheduler.getLogMessages(clauses, fields=('task_id', 'host_id', 'host_name',
+                                                             'msg_ts', 'msg'))
     for log in logs:
         log['time'] = time.asctime(time.localtime(log['msg_ts']))
 
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index e8fb9b0..4ed2ea0 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -487,7 +487,7 @@ class TaskScheduler(object):
         cutoff_ts = time.time() - self.soft_refusal_timeout
         to_drop = []
         for row in get_task_refusals(fields=('id', 'task_id', 'host_id', 'soft', 'ts', 'state')):
-            if ((row['soft'] and row['ts'] < cutoff_ts ) or
+            if ((row['soft'] and row['ts'] < cutoff_ts) or
                     koji.TASK_STATES[row['state']] not in ('FREE', 'OPEN', 'ASSIGNED')):
                 to_drop.append(row['id'])
             else:

From 214d970436e2734c83b0acc01ec0bb42e044a56e Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Sep 26 2023 13:14:48 +0000
Subject: [PATCH 92/92] configurable scheduler parameters


---

diff --git a/kojihub/kojixmlrpc.py b/kojihub/kojixmlrpc.py
index f9000ca..64ff8bc 100644
--- a/kojihub/kojixmlrpc.py
+++ b/kojihub/kojixmlrpc.py
@@ -503,6 +503,15 @@ def load_config(environ):
         ['RPMDefaultChecksums', 'string', 'md5 sha256'],
 
         ['SessionRenewalTimeout', 'integer', 1440],
+
+        # scheduler options
+        ['MaxJobs', 'integer', 15],
+        ['CapacityOvercommit', 'integer', 5],
+        ['ReadyTimeout', 'integer', 180],
+        ['AssignTimeout', 'integer', 300],
+        ['SoftRefusalTimeout', 'integer', 900],
+        ['HostTimeout', 'integer', 900],
+        ['RunInterval', 'integer', 60],
     ]
     opts = {}
     for name, dtype, default in cfgmap:
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
index 4ed2ea0..7718da6 100644
--- a/kojihub/scheduler.py
+++ b/kojihub/scheduler.py
@@ -191,13 +191,13 @@ class TaskScheduler(object):
         self.free_tasks = []
 
         # TODO these things need proper config
-        self.maxjobs = 15  # XXX
-        self.capacity_overcommit = 5
-        self.ready_timeout = 180
-        self.assign_timeout = 300
-        self.soft_refusal_timeout = 900
-        self.host_timeout = 900
-        self.run_interval = 60
+        self.maxjobs = context.opts['MaxJobs']
+        self.capacity_overcommit = context.opts['CapacityOvercommit']
+        self.ready_timeout = context.opts['ReadyTimeout']
+        self.assign_timeout = context.opts['AssignTimeout']
+        self.soft_refusal_timeout = context.opts['SoftRefusalTimeout']
+        self.host_timeout = context.opts['HostTimeout']
+        self.run_interval = context.opts['RunInterval']
 
     def run(self, force=False):
         if not db_lock('scheduler', wait=force):

