From b09fd01677eaf38422db2805ba16254c7a3d0785 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Dec 19 2019 12:55:06 +0000
Subject: additional options to clean database


Fixes: https://pagure.io/koji/issue/1707

---

diff --git a/util/koji-sweep-db b/util/koji-sweep-db
index 265b5c0..6c4283f 100755
--- a/util/koji-sweep-db
+++ b/util/koji-sweep-db
@@ -3,46 +3,163 @@
 import os
 from optparse import OptionParser
 
+from six.moves import xmlrpc_client
 from six.moves.configparser import RawConfigParser
 
 import koji.db
 
 
-def clean_sessions(cursor, vacuum):
-    q = " FROM sessions WHERE update_time < now() - '1 day'::interval"
+def clean_sessions(cursor, vacuum, test, age):
+    q = " FROM sessions WHERE update_time < NOW() - '%s days'::interval" % int(age)
     if options.verbose:
         cursor.execute("SELECT COUNT(*) " + q)
         rows = cursor.fetchall()[0][0]
         print("Deleting %d sessions" % rows)
 
-    cursor.execute("DELETE " + q)
-    if vacuum:
-        cursor.execute("VACUUM ANALYZE sessions")
+    if not test:
+        cursor.execute("DELETE " + q)
+        if vacuum:
+            cursor.execute("VACUUM ANALYZE sessions")
 
 
-def clean_reservations(cursor, vacuum):
-    q = " FROM build_reservations WHERE created < now() - '1 day'::interval"
+def clean_reservations(cursor, vacuum, test, age):
+    q = " FROM build_reservations WHERE created < NOW() - '%s days'::interval" % int(age)
     if options.verbose:
         cursor.execute("SELECT COUNT(*) " + q)
         rows = cursor.fetchall()[0][0]
         print("Deleting %d build reservations" % rows)
 
-    cursor.execute("DELETE " + q)
+    if not test:
+        cursor.execute("DELETE " + q)
+        if vacuum:
+            cursor.execute("VACUUM ANALYZE build_reservations")
+
+
+def clean_notification_tasks(cursor, vacuum, test, age):
+    q = " FROM task WHERE method = 'build' AND completion_time < NOW() - '%s days'::interval" % int(age)
+    if options.verbose:
+        cursor.execute("SELECT COUNT(*) " + q)
+        rows = cursor.fetchall()[0][0]
+        print("Deleting %d tagNotification tasks" % rows)
+
+    if not test:
+        # cascade
+        cursor.execute("DELETE " + q)
+        if vacuum:
+            cursor.execute("VACUUM ANALYZE task")
+
+
+def clean_scratch_tasks(cursor, vacuum, test, age):
+    q = """ FROM task
+              WHERE method = 'build' AND
+              completion_time < NOW()  - '%s days'::interval AND
+              request LIKE '%%%%<name>scratch</name>%%%%'""" % int(age)
+    if options.verbose:
+        cursor.execute("SELECT COUNT(*) " + q)
+        rows = cursor.fetchall()[0][0]
+        print("Deleting %d scratch build tasks" % rows)
+
+    if test:
+        return
+
+    # we should check, that it is really a scratch build
+    ids = []
+    # will be dropped automatically in the end of script/connection
+    cursor.execute("CREATE TEMPORARY TABLE temp_scratch_tasks (task_id INTEGER NOT NULL)")
+    cursor.execute("SELECT id, request " + q)
+    for row in cursor.fetchall():
+        task_id, request = row
+        try:
+            params, method = xmlrpc_client.loads(request)
+            opts = params[2]
+            if opts['scratch']:
+                cursor.execute("INSERT INTO temp_scratch_tasks VALUES (%s)", (task_id,))
+                ids.append(task_id)
+        except:
+            continue
+
+    parents = ids
+    while True:
+        if not parents:
+            break
+        children = []
+        cursor.execute("SELECT id FROM task WHERE parent IN %s", (parents,))
+        for row in cursor.fetchall():
+            children.append(row[0])
+        parents = children
+        if children:
+            values = ', '.join(["(%d)" % task_id for task_id in children])
+            cursor.execute("INSERT INTO temp_scratch_tasks VALUES %s" % values)
+
+    if not ids:
+        return
+
+    # delete standard buildroots
+    cursor.execute("DELETE FROM standard_buildroot WHERE task_id IN (SELECT task_id FROM temp_scratch_tasks)")
+
+    # delete tasks finally
+    cursor.execute("DELETE FROM task WHERE id IN (SELECT task_id FROM temp_scratch_tasks)")
+
     if vacuum:
-        cursor.execute("VACUUM ANALYZE build_reservations")
+        cursor.execute("VACUUM ANALYZE task")
+
+
+def clean_buildroots(cursor, vacuum, test):
+    q = " FROM buildroot WHERE cg_id IS NULL AND id NOT IN (SELECT buildroot_id FROM standard_buildroot)"
+
+    if options.verbose:
+        cursor.execute("SELECT COUNT(*) " + q)
+        rows = cursor.fetchall()[0][0]
+        print("Deleting %d buildroots" % rows)
+
+    if not test:
+        cursor.execute("DELETE FROM buildroot_listing WHERE buildroot_id IN (SELECT id %s)" % q)
+        cursor.execute("DELETE " + q)
+        if vacuum:
+            cursor.execute("VACUUM ANALYZE build_reservations")
 
 
 if __name__ == "__main__":
     global options
     parser = OptionParser("%prog cleans koji database")
     parser.add_option('-v', '--verbose', action="store_true", help="Be verbose")
+    parser.add_option('-t', '--test', action="store_true",
+                      help="Don't delete anything, print estimations (implies -v)")
     parser.add_option('-c', '--conf', default='/etc/koji-hub/hub.conf',
                       action='store', help="Path to koji's hub.conf")
     parser.add_option('--no-vacuum', action="store_false", dest="vacuum",
                       default=True,
                       help="Don't run vacuum on affected tables")
+    parser.add_option('--sessions-age', type=int,
+                      action="store", default=1, metavar="DAYS",
+                      help="Delete sessions older than this (default: 1 day)")
+    parser.add_option('--reservations-age', type=int,
+                      action="store", default=1, metavar="DAYS",
+                      help="Delete CG reservations older than this (default: 1 day)")
+    parser.add_option('--tag-notifications', action="store_true",
+                      help="Delete tagNotification tasks", default=False)
+    parser.add_option('--tag-notifications-age', type=int,
+                      action="store", default=730, metavar="DAYS",
+                      help="Delete tagNotification tasks older than this (default: 2 years)")
+    parser.add_option("--scratch-builds", action="store_true",
+                      dest="scratch", default=False,
+                      help="Delete scratch build tasks")
+    parser.add_option('--scratch-builds-age', type=int, dest="scratch_age",
+                      action="store", default=730, metavar="DAYS",
+                      help="Delete scratch builds' tasks older than this (default: 2 years")
+    parser.add_option('--buildroots', action="store_true",
+                      help="Delete unreferenced buildroots")
+    parser.add_option('-f', '--force', action="store_true",
+                      help="This is needs for all options affecting auditable data")
     options, args = parser.parse_args()
 
+    if options.test:
+        options.verbose = True
+        print("Running in test mode, no changes will be made")
+
+    if not options.force and (options.tag_notifications or options.scratch or options.buildroots):
+        parser.error("You need to pass --force to trigger these actions")
+
     if not os.path.exists(options.conf):
         parser.error("Config file doesn't exist")
 
@@ -85,6 +202,11 @@ if __name__ == "__main__":
     conn.set_session(autocommit=True)
     cursor = conn.cursor()
 
-    clean_sessions(cursor, options.vacuum)
-    clean_reservations(cursor, options.vacuum)
-    conn.commit()
+    clean_sessions(cursor, options.vacuum, options.test, options.sessions_age)
+    clean_reservations(cursor, options.vacuum, options.test, options.reservations_age)
+    if options.tag_notifications:
+        clean_notification_tasks(cursor, options.vacuum, options.test, age=options.tag_notifications_age)
+    if options.scratch:
+        clean_scratch_tasks(cursor, options.vacuum, options.test, age=options.scratch_age)
+    if options.buildroots:
+        clean_buildroots(cursor, options.vacuum, options.test)

