From 7a03fce17ce82c5ce8f9f023ab1aafcb04139218 Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Sep 10 2021 01:12:03 +0000 Subject: [PATCH 1/11] kojira image changes --- diff --git a/images/kojira/Dockerfile b/images/kojira/Dockerfile index f927ef5..bac29b5 100644 --- a/images/kojira/Dockerfile +++ b/images/kojira/Dockerfile @@ -1,6 +1,5 @@ -FROM centos:8 +FROM fedora:34 -RUN yum install -y epel-release RUN yum install -y koji-utils COPY entrypoint.sh /entrypoint.sh From 408dc72861f82610bf3af549e800dc8330d4a366 Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Sep 10 2021 01:12:04 +0000 Subject: [PATCH 2/11] koji-builder image changes --- diff --git a/images/koji-builder/Dockerfile b/images/koji-builder/Dockerfile index b100e26..df8fbc9 100644 --- a/images/koji-builder/Dockerfile +++ b/images/koji-builder/Dockerfile @@ -1,7 +1,6 @@ -FROM centos:8 +FROM fedora:34 -RUN yum install -y epel-release && \ -yum install -y koji-builder +RUN yum install -y koji-builder COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh && \ @@ -11,4 +10,4 @@ ENV USER=1001 ENTRYPOINT bash /entrypoint.sh -USER 1001 \ No newline at end of file +USER 1001 From 7e4059b1ea6d54773acfc3e7605cc86fa84f4c5a Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Sep 10 2021 01:12:04 +0000 Subject: [PATCH 3/11] koji-hub image changes --- diff --git a/images/koji-hub/Dockerfile b/images/koji-hub/Dockerfile index ec9124f..64cec29 100644 --- a/images/koji-hub/Dockerfile +++ b/images/koji-hub/Dockerfile @@ -1,6 +1,5 @@ -FROM centos:8 +FROM fedora:34 -RUN yum install -y epel-release RUN yum install -y \ koji koji-hub koji-web koji-hub-plugins \ curl httpd mod_ssl postgresql fedora-messaging python3-mod_wsgi From 1f796fce0003ecf917f8376a0f710e993dab6538 Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Sep 10 2021 01:12:04 +0000 Subject: [PATCH 4/11] operator image changes --- diff --git a/operator/Dockerfile b/operator/Dockerfile index 574e762..5e95dd0 100644 --- a/operator/Dockerfile +++ b/operator/Dockerfile @@ -1,20 +1,11 @@ -# FROM quay.io/operator-framework/ansible-operator:v1.7.2 - -# COPY requirements.yml ${HOME}/requirements.yml -# RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \ -# && chmod -R ug+rwx ${HOME}/.ansible - -# COPY watches.yaml ${HOME}/watches.yaml -# COPY roles/ ${HOME}/roles/ -# COPY playbooks/ ${HOME}/playbooks/ FROM quay.io/operator-framework/ansible-operator:v1.7.2 USER root -RUN dnf install gcc libpq libpq-devel python38-devel krb5-devel wget -y -# RUN wget https://bootstrap.pypa.io/get-pip.py && \ -# python3.8 get-pip.py && \ -# rm get-pip.py -RUN pip3.8 install psycopg2 koji +RUN dnf install -y gcc libpq libpq-devel python38-devel krb5-devel wget python38-psycopg2 +#TODO: find a better way to install koji and its schema.sql file +RUN pip3.8 install koji==1.25.1 +COPY hack/schema.sql /usr/share/doc/koji/docs/schema.sql +RUN chmod 644 /usr/share/doc/koji/docs/schema.sql USER ${USER_ID} COPY requirements.yml ${HOME}/requirements.yml diff --git a/operator/hack/schema.sql b/operator/hack/schema.sql new file mode 100644 index 0000000..d9eca28 --- /dev/null +++ b/operator/hack/schema.sql @@ -0,0 +1,953 @@ + +-- vim:et:sw=8 + +BEGIN WORK; + +-- We use the events table to sequence time +-- in the event that the system clock rolls back, event_ids will retain proper sequencing +CREATE TABLE events ( + id SERIAL NOT NULL PRIMARY KEY, + time TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +) WITHOUT OIDS; + +-- A function that creates an event and returns the id, used as DEFAULT value for versioned tables +CREATE FUNCTION get_event() RETURNS INTEGER AS ' + INSERT INTO events (time) VALUES (clock_timestamp()); + SELECT currval(''events_id_seq'')::INTEGER; +' LANGUAGE SQL; + +-- A convenience function for converting events to timestamps, useful for +-- quick queries where you want to avoid JOINs. +CREATE FUNCTION get_event_time(INTEGER) RETURNS TIMESTAMPTZ AS ' + SELECT time FROM events WHERE id=$1; +' LANGUAGE SQL; + +-- this table is used to label events +-- most events will be unlabeled, so keeping this separate saves space +CREATE TABLE event_labels ( + event_id INTEGER NOT NULL REFERENCES events(id), + label VARCHAR(255) UNIQUE NOT NULL +) WITHOUT OIDS; + + +-- User and session data +CREATE TABLE users ( + id SERIAL NOT NULL PRIMARY KEY, + name VARCHAR(255) UNIQUE NOT NULL, + password VARCHAR(255), + status INTEGER NOT NULL, + usertype INTEGER NOT NULL +) WITHOUT OIDS; + +CREATE TABLE user_krb_principals ( + user_id INTEGER NOT NULL REFERENCES users(id), + krb_principal VARCHAR(255) NOT NULL UNIQUE, + PRIMARY KEY (user_id, krb_principal) +) WITHOUT OIDS; + +CREATE TABLE permissions ( + id SERIAL NOT NULL PRIMARY KEY, + name VARCHAR(50) UNIQUE NOT NULL +) WITHOUT OIDS; + +-- Some basic perms +INSERT INTO permissions (name) VALUES ('admin'); +INSERT INTO permissions (name) VALUES ('appliance'); +INSERT INTO permissions (name) VALUES ('build'); +INSERT INTO permissions (name) VALUES ('dist-repo'); +INSERT INTO permissions (name) VALUES ('host'); +INSERT INTO permissions (name) VALUES ('image'); +INSERT INTO permissions (name) VALUES ('image-import'); +INSERT INTO permissions (name) VALUES ('livecd'); +INSERT INTO permissions (name) VALUES ('maven-import'); +INSERT INTO permissions (name) VALUES ('repo'); +INSERT INTO permissions (name) VALUES ('sign'); +INSERT INTO permissions (name) VALUES ('tag'); +INSERT INTO permissions (name) VALUES ('target'); +INSERT INTO permissions (name) VALUES ('win-admin'); +INSERT INTO permissions (name) VALUES ('win-import'); + +CREATE TABLE user_perms ( + user_id INTEGER NOT NULL REFERENCES users(id), + perm_id INTEGER NOT NULL REFERENCES permissions(id), +-- versioned - see VERSIONING + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, user_id, perm_id), + UNIQUE (user_id,perm_id,active) +) WITHOUT OIDS; + +-- groups are represented as users w/ usertype=2 +CREATE TABLE user_groups ( + user_id INTEGER NOT NULL REFERENCES users(id), + group_id INTEGER NOT NULL REFERENCES users(id), +-- versioned - see VERSIONING + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, user_id, group_id), + UNIQUE (user_id,group_id,active) +) WITHOUT OIDS; + +-- a session can create subsessions, which are just new sessions whose +-- 'master' field points back to the session. This field should +-- always point to the top session. If the master session is expired, +-- the all its subsessions should be expired as well. +-- If a session is exclusive, it is the only session allowed for its +-- user. The 'exclusive' field is either NULL or TRUE, never FALSE. This +-- is so exclusivity can be enforced with a unique condition. +CREATE TABLE sessions ( + id SERIAL NOT NULL PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + expired BOOLEAN NOT NULL DEFAULT FALSE, + master INTEGER REFERENCES sessions(id), + key VARCHAR(255), + authtype INTEGER, + hostip VARCHAR(255), + callnum INTEGER, + start_time TIMESTAMPTZ NOT NULL DEFAULT NOW(), + update_time TIMESTAMPTZ NOT NULL DEFAULT NOW(), + exclusive BOOLEAN CHECK (exclusive), + CONSTRAINT no_exclusive_subsessions CHECK ( + master IS NULL OR "exclusive" IS NULL), + CONSTRAINT exclusive_expired_sane CHECK ( + expired IS FALSE OR "exclusive" IS NULL), + UNIQUE (user_id,exclusive) +) WITHOUT OIDS; +CREATE INDEX sessions_master ON sessions(master); +CREATE INDEX sessions_active_and_recent ON sessions(expired, master, update_time) WHERE (expired = FALSE AND master IS NULL); +CREATE INDEX sessions_expired ON sessions(expired); + +-- Channels are used to limit which tasks are run on which machines. +-- Each task is assigned to a channel and each host 'listens' on one +-- or more channels. A host will only accept tasks for channels it is +-- listening to. +CREATE TABLE channels ( + id SERIAL NOT NULL PRIMARY KEY, + name VARCHAR(128) UNIQUE NOT NULL +) WITHOUT OIDS; + +-- create default channel +INSERT INTO channels (name) VALUES ('default'); +INSERT INTO channels (name) VALUES ('createrepo'); +INSERT INTO channels (name) VALUES ('maven'); +INSERT INTO channels (name) VALUES ('livecd'); +INSERT INTO channels (name) VALUES ('appliance'); +INSERT INTO channels (name) VALUES ('vm'); +INSERT INTO channels (name) VALUES ('image'); +INSERT INTO channels (name) VALUES ('livemedia'); + +-- Here we track the build machines +-- each host has an entry in the users table also +-- capacity: the hosts weighted task capacity +CREATE TABLE host ( + id SERIAL NOT NULL PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users (id), + name VARCHAR(128) UNIQUE NOT NULL, + task_load FLOAT CHECK (NOT task_load < 0) NOT NULL DEFAULT 0.0, + ready BOOLEAN NOT NULL DEFAULT 'false' +) WITHOUT OIDS; + +CREATE TABLE host_config ( + host_id INTEGER NOT NULL REFERENCES host(id), + arches TEXT, + capacity FLOAT CHECK (capacity > 1) NOT NULL DEFAULT 2.0, + description TEXT, + comment TEXT, + enabled BOOLEAN NOT NULL DEFAULT 'true', +-- versioned - see desc above + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, host_id), + UNIQUE (host_id, active) +) WITHOUT OIDS; +CREATE INDEX host_config_by_active_and_enabled ON host_config(active, enabled); + +CREATE TABLE host_channels ( + host_id INTEGER NOT NULL REFERENCES host(id), + channel_id INTEGER NOT NULL REFERENCES channels(id), +-- versioned - see desc above + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, host_id, channel_id), + UNIQUE (host_id, channel_id, active) +) WITHOUT OIDS; + + +-- tasks are pretty general and may refer to all sorts of jobs, not +-- just package builds. +-- tasks may spawn subtasks (hence the parent field) +-- top-level tasks have NULL parent +-- the request and result fields are xmlrpc data. +-- this means each task is effectively an xmlrpc call, using this table as +-- the medium. +-- the host_id field indicates which host is running the task. This field +-- is used to lock the task. +-- weight: the weight of the task (vs. host capacity) +-- label: this field is used to label subtasks. top-level tasks will not +-- have a label. some subtasks may be unlabeled. labels are used in task +-- failover to prevent duplication of work. +CREATE TABLE task ( + id SERIAL NOT NULL PRIMARY KEY, + state INTEGER, + create_time TIMESTAMPTZ NOT NULL DEFAULT NOW(), + start_time TIMESTAMPTZ, + completion_time TIMESTAMPTZ, + channel_id INTEGER NOT NULL REFERENCES channels(id), + host_id INTEGER REFERENCES host (id), + parent INTEGER REFERENCES task (id), + label VARCHAR(255), + waiting BOOLEAN, + awaited BOOLEAN, + owner INTEGER REFERENCES users(id) NOT NULL, + method TEXT, + request TEXT, + result TEXT, + eta INTEGER, + arch VARCHAR(16) NOT NULL, + priority INTEGER, + weight FLOAT CHECK (NOT weight < 0) NOT NULL DEFAULT 1.0, + CONSTRAINT parent_label_sane CHECK ( + parent IS NOT NULL OR label IS NULL), + UNIQUE (parent,label) +) WITHOUT OIDS; + +CREATE INDEX task_by_state ON task (state); +-- CREATE INDEX task_by_parent ON task (parent); (unique condition creates similar index) +CREATE INDEX task_by_host ON task (host_id); +CREATE INDEX task_by_no_parent_state_method ON task(parent, state, method) WHERE parent IS NULL; + + +-- by package, we mean srpm +-- we mean the package in general, not an individual build +CREATE TABLE package ( + id SERIAL NOT NULL PRIMARY KEY, + name TEXT UNIQUE NOT NULL +) WITHOUT OIDS; + +-- CREATE INDEX package_by_name ON package (name); +-- (implicitly created by unique constraint) + + +CREATE TABLE volume ( + id SERIAL NOT NULL PRIMARY KEY, + name TEXT UNIQUE NOT NULL +) WITHOUT OIDS; + +INSERT INTO volume (id, name) VALUES (0, 'DEFAULT'); + +-- data for content generators +CREATE TABLE content_generator ( + id SERIAL PRIMARY KEY, + name TEXT UNIQUE NOT NULL +) WITHOUT OIDS; + +-- here we track the built packages +-- this is at the srpm level, since builds are by srpm +-- see rpminfo for isolated packages +-- even though we track epoch, we demand that N-V-R be unique +-- task_id: a reference to the task creating the build, may be +-- null, or may point to a deleted task. +CREATE TABLE build ( + id SERIAL NOT NULL PRIMARY KEY, + volume_id INTEGER NOT NULL REFERENCES volume (id), + pkg_id INTEGER NOT NULL REFERENCES package (id) DEFERRABLE, + version TEXT NOT NULL, + release TEXT NOT NULL, + epoch INTEGER, + source TEXT, + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + start_time TIMESTAMPTZ, + completion_time TIMESTAMPTZ, + state INTEGER NOT NULL, + task_id INTEGER REFERENCES task (id), + owner INTEGER NOT NULL REFERENCES users (id), + cg_id INTEGER REFERENCES content_generator(id), + extra TEXT, + CONSTRAINT build_pkg_ver_rel UNIQUE (pkg_id, version, release), + CONSTRAINT completion_sane CHECK ((state = 0 AND completion_time IS NULL) OR + (state != 0 AND completion_time IS NOT NULL)) +) WITHOUT OIDS; + +CREATE INDEX build_by_pkg_id ON build (pkg_id); +CREATE INDEX build_completion ON build(completion_time); + + +CREATE TABLE btype ( + id SERIAL NOT NULL PRIMARY KEY, + name TEXT UNIQUE NOT NULL +) WITHOUT OIDS; + + +-- legacy build types +INSERT INTO btype(name) VALUES ('rpm'); +INSERT INTO btype(name) VALUES ('maven'); +INSERT INTO btype(name) VALUES ('win'); +INSERT INTO btype(name) VALUES ('image'); + + +CREATE TABLE build_types ( + build_id INTEGER NOT NULL REFERENCES build(id), + btype_id INTEGER NOT NULL REFERENCES btype(id), + PRIMARY KEY (build_id, btype_id) +) WITHOUT OIDS; + + +-- Note: some of these CREATEs may seem a little out of order. This is done to keep +-- the references sane. + +CREATE TABLE tag ( + id SERIAL NOT NULL PRIMARY KEY, + name TEXT UNIQUE NOT NULL +) WITHOUT OIDS; + +-- CREATE INDEX tag_by_name ON tag (name); +-- (implicitly created by unique constraint) + + +-- VERSIONING +-- Several tables are versioned with the following scheme. Since this +-- is the first, here is the explanation of how it works. +-- The versioning fields are: create_event, revoke_event, and active +-- The active field is either True or NULL, it is never False! +-- The create_event and revoke_event fields refer to the event table +-- A version is active if active is not NULL +-- (an active version also has NULL revoke_event.) +-- A UNIQUE condition can incorporate the 'active' field, making it +-- apply only to the active versions. +-- When a version is made inactive (revoked): +-- revoke_event is set +-- active is set to NULL +-- Query for current data with WHERE active is not NULL +-- (should be same as WHERE revoke_event is NULL) +-- Query for data at event e with WHERE create_event <= e AND e < revoke_event +CREATE TABLE tag_inheritance ( + tag_id INTEGER NOT NULL REFERENCES tag(id), + parent_id INTEGER NOT NULL REFERENCES tag(id), + priority INTEGER NOT NULL, + maxdepth INTEGER, + intransitive BOOLEAN NOT NULL DEFAULT 'false', + noconfig BOOLEAN NOT NULL DEFAULT 'false', + pkg_filter TEXT, +-- versioned - see desc above + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, tag_id, priority), + UNIQUE (tag_id,priority,active), + UNIQUE (tag_id,parent_id,active) +) WITHOUT OIDS; + +CREATE INDEX tag_inheritance_by_parent ON tag_inheritance (parent_id); + +-- XXX - need more config options listed here +-- perm_id: the permission that is required to apply the tag. can be NULL +-- +CREATE TABLE tag_config ( + tag_id INTEGER NOT NULL REFERENCES tag(id), + arches TEXT, + perm_id INTEGER REFERENCES permissions(id), + locked BOOLEAN NOT NULL DEFAULT 'false', + maven_support BOOLEAN NOT NULL DEFAULT FALSE, + maven_include_all BOOLEAN NOT NULL DEFAULT FALSE, +-- versioned - see desc above + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, tag_id), + UNIQUE (tag_id,active) +) WITHOUT OIDS; + +CREATE TABLE tag_extra ( + tag_id INTEGER NOT NULL REFERENCES tag(id), + key TEXT NOT NULL, + value TEXT, +-- versioned - see desc above + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, tag_id, key), + UNIQUE (tag_id, key, active) +) WITHOUT OIDS; + +-- the tag_updates table provides a mechanism to indicate changes relevant to tag +-- that are not reflected in a versioned table. For example: builds changing volumes, +-- changes to external repo content, additional rpms imported to an existing build +CREATE TABLE tag_updates ( + id SERIAL NOT NULL PRIMARY KEY, + tag_id INTEGER NOT NULL REFERENCES tag(id), + update_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + updater_id INTEGER NOT NULL REFERENCES users(id), + update_type INTEGER NOT NULL +) WITHOUT OIDS; + +CREATE INDEX tag_updates_by_tag ON tag_updates (tag_id); +CREATE INDEX tag_updates_by_event ON tag_updates (update_event); + +-- a build target tells the system where to build the package +-- and how to tag it afterwards. +CREATE TABLE build_target ( + id SERIAL NOT NULL PRIMARY KEY, + name TEXT UNIQUE NOT NULL +) WITHOUT OIDS; + + +CREATE TABLE build_target_config ( + build_target_id INTEGER NOT NULL REFERENCES build_target(id), + build_tag INTEGER NOT NULL REFERENCES tag(id), + dest_tag INTEGER NOT NULL REFERENCES tag(id), +-- versioned - see desc above + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, build_target_id), + UNIQUE (build_target_id,active) +) WITHOUT OIDS; + + +-- track repos +CREATE TABLE repo ( + id SERIAL NOT NULL PRIMARY KEY, + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + tag_id INTEGER NOT NULL REFERENCES tag(id), + state INTEGER, + dist BOOLEAN DEFAULT 'false', + task_id INTEGER NULL REFERENCES task(id) +) WITHOUT OIDS; + +-- external yum repos +create table external_repo ( + id SERIAL NOT NULL PRIMARY KEY, + name TEXT UNIQUE NOT NULL +); +-- fake repo id for internal stuff (needed for unique index) +INSERT INTO external_repo (id, name) VALUES (0, 'INTERNAL'); + +CREATE TABLE external_repo_config ( + external_repo_id INTEGER NOT NULL REFERENCES external_repo(id), + url TEXT NOT NULL, +-- versioned - see earlier description of versioning + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, external_repo_id), + UNIQUE (external_repo_id, active) +) WITHOUT OIDS; + +CREATE TABLE tag_external_repos ( + tag_id INTEGER NOT NULL REFERENCES tag(id), + external_repo_id INTEGER NOT NULL REFERENCES external_repo(id), + priority INTEGER NOT NULL, + merge_mode TEXT NOT NULL DEFAULT 'koji', + arches TEXT, +-- versioned - see earlier description of versioning + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, tag_id, priority), + UNIQUE (tag_id, priority, active), + UNIQUE (tag_id, external_repo_id, active) +); + +CREATE TABLE cg_users ( + cg_id INTEGER NOT NULL REFERENCES content_generator (id), + user_id INTEGER NOT NULL REFERENCES users (id), +-- versioned - see earlier description of versioning + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, cg_id, user_id), + UNIQUE (cg_id, user_id, active) +) WITHOUT OIDS; + +CREATE TABLE build_reservations ( + build_id INTEGER NOT NULL REFERENCES build(id), + token VARCHAR(64), + created TIMESTAMPTZ NOT NULL, + PRIMARY KEY (build_id) +) WITHOUT OIDS; +CREATE INDEX build_reservations_created ON build_reservations(created); + +-- here we track the buildroots on the machines +CREATE TABLE buildroot ( + id SERIAL NOT NULL PRIMARY KEY, + br_type INTEGER NOT NULL, + cg_id INTEGER REFERENCES content_generator (id), + cg_version TEXT, + CONSTRAINT cg_sane CHECK ( + (cg_id IS NULL AND cg_version IS NULL) + OR (cg_id IS NOT NULL AND cg_version IS NOT NULL)), + container_type TEXT, + container_arch TEXT, + CONSTRAINT container_sane CHECK ( + (container_type IS NULL AND container_arch IS NULL) + OR (container_type IS NOT NULL AND container_arch IS NOT NULL)), + host_os TEXT, + host_arch TEXT, + extra TEXT +) WITHOUT OIDS; + +CREATE TABLE standard_buildroot ( + buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), + host_id INTEGER NOT NULL REFERENCES host(id), + repo_id INTEGER NOT NULL REFERENCES repo (id), + task_id INTEGER NOT NULL REFERENCES task (id), + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + retire_event INTEGER, + state INTEGER +) WITHOUT OIDS; + +CREATE TABLE buildroot_tools_info ( + buildroot_id INTEGER NOT NULL REFERENCES buildroot(id), + tool TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (buildroot_id, tool) +) WITHOUT OIDS; + + +-- track spun images (livecds, installation, VMs...) +CREATE TABLE image_builds ( + build_id INTEGER NOT NULL PRIMARY KEY REFERENCES build(id) +) WITHOUT OIDS; + +-- this table associates tags with builds. an entry here tags a package +CREATE TABLE tag_listing ( + build_id INTEGER NOT NULL REFERENCES build (id), + tag_id INTEGER NOT NULL REFERENCES tag (id), +-- versioned - see earlier description of versioning + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, build_id, tag_id), + UNIQUE (build_id,tag_id,active) +) WITHOUT OIDS; +CREATE INDEX tag_listing_tag_id_key ON tag_listing(tag_id); + +-- this is a per-tag list of packages, with some extra info +-- so this allows you to explicitly state which packages belong where +-- (as opposed to beehive where this can only be done at the collection level) +-- these are packages in general, not specific builds. +-- this list limits which builds can be tagged with which tags +-- if blocked is true, then the package is specifically not included. this +-- prevents the package from being included via inheritance +CREATE TABLE tag_packages ( + package_id INTEGER NOT NULL REFERENCES package (id), + tag_id INTEGER NOT NULL REFERENCES tag (id), + blocked BOOLEAN NOT NULL DEFAULT FALSE, + extra_arches TEXT, +-- versioned - see earlier description of versioning + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, package_id, tag_id), + UNIQUE (package_id,tag_id,active) +) WITHOUT OIDS; +CREATE INDEX tag_packages_active_tag_id ON tag_packages(active, tag_id); +CREATE INDEX tag_packages_create_event ON tag_packages(create_event); +CREATE INDEX tag_packages_revoke_event ON tag_packages(revoke_event); + +CREATE TABLE tag_package_owners ( + package_id INTEGER NOT NULL REFERENCES package(id), + tag_id INTEGER NOT NULL REFERENCES tag (id), + owner INTEGER NOT NULL REFERENCES users(id), +-- versioned - see earlier description of versioning + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, package_id, tag_id), + UNIQUE (package_id,tag_id,active) +) WITHOUT OIDS; + +-- package groups (per tag). used for generating comps for the tag repos +CREATE TABLE groups ( + id SERIAL NOT NULL PRIMARY KEY, + name VARCHAR(50) UNIQUE NOT NULL + -- corresponds to the id field in a comps group +) WITHOUT OIDS; + +-- if blocked is true, then the group is specifically not included. this +-- prevents the group from being included via inheritance +CREATE TABLE group_config ( + group_id INTEGER NOT NULL REFERENCES groups (id), + tag_id INTEGER NOT NULL REFERENCES tag (id), + blocked BOOLEAN NOT NULL DEFAULT FALSE, + exported BOOLEAN DEFAULT TRUE, + display_name TEXT NOT NULL, + is_default BOOLEAN, + uservisible BOOLEAN, + description TEXT, + langonly TEXT, + biarchonly BOOLEAN, +-- versioned - see earlier description of versioning + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, group_id, tag_id), + UNIQUE (group_id,tag_id,active) +) WITHOUT OIDS; + +CREATE TABLE group_req_listing ( + group_id INTEGER NOT NULL REFERENCES groups (id), + tag_id INTEGER NOT NULL REFERENCES tag (id), + req_id INTEGER NOT NULL REFERENCES groups (id), + blocked BOOLEAN NOT NULL DEFAULT FALSE, + type VARCHAR(25), + is_metapkg BOOLEAN NOT NULL DEFAULT FALSE, +-- versioned - see earlier description of versioning + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, group_id, tag_id, req_id), + UNIQUE (group_id,tag_id,req_id,active) +) WITHOUT OIDS; + +-- if blocked is true, then the package is specifically not included. this +-- prevents the package from being included in the group via inheritance +-- package refers to an rpm name, not necessarily an srpm name (so it does +-- not reference the package table). +CREATE TABLE group_package_listing ( + group_id INTEGER NOT NULL REFERENCES groups (id), + tag_id INTEGER NOT NULL REFERENCES tag (id), + package TEXT, + blocked BOOLEAN NOT NULL DEFAULT FALSE, + type VARCHAR(25) NOT NULL, + basearchonly BOOLEAN, + requires TEXT, +-- versioned - see earlier description of versioning + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, group_id, tag_id, package), + UNIQUE (group_id,tag_id,package,active) +) WITHOUT OIDS; + +-- rpminfo tracks individual rpms (incl srpms) +-- buildroot_id can be NULL (for externally built packages) +-- even though we track epoch, we demand that N-V-R.A be unique +-- we don't store filename b/c filename should be N-V-R.A.rpm +CREATE TABLE rpminfo ( + id SERIAL NOT NULL PRIMARY KEY, + build_id INTEGER REFERENCES build (id), + buildroot_id INTEGER REFERENCES buildroot (id), + name TEXT NOT NULL, + version TEXT NOT NULL, + release TEXT NOT NULL, + epoch INTEGER, + arch VARCHAR(16) NOT NULL, + external_repo_id INTEGER NOT NULL REFERENCES external_repo(id), + payloadhash TEXT NOT NULL, + size BIGINT NOT NULL, + buildtime BIGINT NOT NULL, + metadata_only BOOLEAN NOT NULL DEFAULT FALSE, + extra TEXT, + CONSTRAINT rpminfo_unique_nvra UNIQUE (name,version,release,arch,external_repo_id) +) WITHOUT OIDS; +CREATE INDEX rpminfo_build ON rpminfo(build_id); + +-- sighash is the checksum of the signature header +CREATE TABLE rpmsigs ( + rpm_id INTEGER NOT NULL REFERENCES rpminfo (id), + sigkey TEXT NOT NULL, + sighash TEXT NOT NULL, + CONSTRAINT rpmsigs_no_resign UNIQUE (rpm_id, sigkey) +) WITHOUT OIDS; + +-- buildroot_listing needs to be created after rpminfo so it can reference it +CREATE TABLE buildroot_listing ( + buildroot_id INTEGER NOT NULL REFERENCES buildroot(id), + rpm_id INTEGER NOT NULL REFERENCES rpminfo(id), + is_update BOOLEAN NOT NULL DEFAULT FALSE, + UNIQUE (buildroot_id,rpm_id) +) WITHOUT OIDS; +CREATE INDEX buildroot_listing_rpms ON buildroot_listing(rpm_id); + +CREATE TABLE build_notifications ( + id SERIAL NOT NULL PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users (id), + package_id INTEGER REFERENCES package (id), + tag_id INTEGER REFERENCES tag (id), + success_only BOOLEAN NOT NULL DEFAULT FALSE, + email TEXT NOT NULL +) WITHOUT OIDS; + +CREATE TABLE build_notifications_block ( + id SERIAL NOT NULL PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users (id), + package_id INTEGER REFERENCES package (id), + tag_id INTEGER REFERENCES tag (id) +) WITHOUT OIDS; + +GRANT SELECT ON build, package, task, tag, +tag_listing, tag_config, tag_inheritance, tag_packages, +rpminfo TO PUBLIC; + +-- example code to add initial admins +-- insert into users (name, usertype, status, krb_principal) values ('admin', 0, 0, 'admin@EXAMPLE.COM'); +-- insert into user_perms (user_id, perm_id) +-- select users.id, permissions.id from users, permissions +-- where users.name in ('admin') +-- and permissions.name = 'admin'; + +-- Schema additions for multiplatform support + +-- we need to track some additional metadata about Maven builds +CREATE TABLE maven_builds ( + build_id INTEGER NOT NULL PRIMARY KEY REFERENCES build(id), + group_id TEXT NOT NULL, + artifact_id TEXT NOT NULL, + version TEXT NOT NULL +) WITHOUT OIDS; + +-- Windows-specific build information +CREATE TABLE win_builds ( + build_id INTEGER NOT NULL PRIMARY KEY REFERENCES build(id), + platform TEXT NOT NULL +) WITHOUT OIDS; + +-- Even though we call this archiveinfo, we can probably use it for +-- any filetype output by a build process. In general they will be +-- archives (.zip, .jar, .tar.gz) but could also be installer executables (.exe) +CREATE TABLE archivetypes ( + id SERIAL NOT NULL PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + description TEXT NOT NULL, + extensions TEXT NOT NULL +) WITHOUT OIDS; + +INSERT INTO archivetypes (name, description, extensions) VALUES ('jar', 'Jar file', 'jar war rar ear sar jdocbook jdocbook-style'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('zip', 'Zip file', 'zip'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('pom', 'Maven Project Object Management file', 'pom'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('tar', 'Tar file', 'tar tar.gz tar.bz2 tar.xz tgz'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('xml', 'XML file', 'xml'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('xmlcompressed', 'Compressed XML file', 'xml.gz xml.bz2 xml.xz'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('xsd', 'XML Schema Definition', 'xsd'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('spec', 'RPM spec file', 'spec'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('exe', 'Windows executable', 'exe'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('dll', 'Windows dynamic link library', 'dll'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('lib', 'Windows import library', 'lib'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('sys', 'Windows device driver', 'sys'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('inf', 'Windows driver information file', 'inf'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('cat', 'Windows catalog file', 'cat'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('msi', 'Windows Installer package', 'msi'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('pdb', 'Windows debug information', 'pdb'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('oem', 'Windows driver oem file', 'oem'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('iso', 'CD/DVD Image', 'iso'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('raw', 'Raw disk image', 'raw'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('qcow', 'QCOW image', 'qcow'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('qcow2', 'QCOW2 image', 'qcow2'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('vmdk', 'vSphere image', 'vmdk'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('ova', 'Open Virtualization Archive', 'ova'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('ks', 'Kickstart', 'ks'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('cfg', 'Configuration file', 'cfg'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('vdi', 'VirtualBox Virtual Disk Image', 'vdi'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('aar', 'Binary distribution of an Android Library project', 'aar'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('apklib', 'Source distribution of an Android Library project', 'apklib'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('cab', 'Windows cabinet file', 'cab'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('dylib', 'OS X dynamic library', 'dylib'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('gem', 'Ruby gem', 'gem'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('ini', 'INI config file', 'ini'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('js', 'Javascript file', 'js'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('ldif', 'LDAP Data Interchange Format file', 'ldif'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('manifest', 'Runtime environment for .NET applications', 'manifest'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('msm', 'Windows merge module', 'msm'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('properties', 'Properties file', 'properties'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('sig', 'Signature file', 'sig signature'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('so', 'Shared library', 'so'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('txt', 'Text file', 'txt'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('vhd', 'Hyper-V image', 'vhd'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('vhdx', 'Hyper-V Virtual Hard Disk v2 image', 'vhdx'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('wsf', 'Windows script file', 'wsf'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('box', 'Vagrant Box Image', 'box'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('raw-xz', 'xz compressed raw disk image', 'raw.xz'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('json', 'JSON data', 'json'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('key', 'Key file', 'key'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('dot', 'DOT graph description', 'dot gv'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('groovy', 'Groovy script file', 'groovy gvy'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('batch', 'Batch file', 'bat'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('shell', 'Shell script', 'sh'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('rc', 'Resource file', 'rc'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('wsdl', 'Web Services Description Language', 'wsdl'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('obr', 'OSGi Bundle Repository', 'obr'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('liveimg-squashfs', 'liveimg compatible squashfs image', 'liveimg.squashfs'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('tlb', 'OLE type library file', 'tlb'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('jnilib', 'Java Native Interface library', 'jnilib'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('yaml', 'YAML Ain''t Markup Language', 'yaml yml'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('xjb', 'JAXB(Java Architecture for XML Binding) Binding Customization File', 'xjb'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('raw-gz', 'GZIP compressed raw disk image', 'raw.gz'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('qcow2-compressed', 'Compressed QCOW2 image', 'qcow2.gz qcow2.xz'); +-- add compressed iso-compressed, vhd-compressed, vhdx-compressed, and vmdk-compressed: From schema-upgrade-1.18-1.19 +INSERT INTO archivetypes (name, description, extensions) VALUES ('iso-compressed', 'Compressed iso image', 'iso.gz iso.xz'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('vhd-compressed', 'Compressed VHD image', 'vhd.gz vhd.xz'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('vhdx-compressed', 'Compressed VHDx image', 'vhd.gz vhd.xz'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('vmdk-compressed', 'Compressed VMDK image', 'vmdk.gz vmdk.xz'); +-- add kernel-image and imitramfs: From schema-upgrade-1.18-1.19 +INSERT INTO archivetypes (name, description, extensions) VALUES ('kernel-image', 'Kernel BZ2 Image', 'vmlinuz vmlinuz.gz vmlinuz.xz'); +INSERT INTO archivetypes (name, description, extensions) VALUES ('initramfs', 'Compressed Initramfs Image', 'img'); + + +-- Do we want to enforce a constraint that a build can only generate one +-- archive with a given name? +CREATE TABLE archiveinfo ( + id SERIAL NOT NULL PRIMARY KEY, + type_id INTEGER NOT NULL REFERENCES archivetypes (id), + btype_id INTEGER REFERENCES btype(id), + -- ^ TODO add NOT NULL + build_id INTEGER NOT NULL REFERENCES build (id), + buildroot_id INTEGER REFERENCES buildroot (id), + filename TEXT NOT NULL, + size BIGINT NOT NULL, + checksum TEXT NOT NULL, + checksum_type INTEGER NOT NULL, + metadata_only BOOLEAN NOT NULL DEFAULT FALSE, + extra TEXT +) WITHOUT OIDS; +CREATE INDEX archiveinfo_build_idx ON archiveinfo (build_id); +CREATE INDEX archiveinfo_buildroot_idx on archiveinfo (buildroot_id); +CREATE INDEX archiveinfo_type_idx on archiveinfo (type_id); +CREATE INDEX archiveinfo_filename_idx on archiveinfo(filename); + +CREATE TABLE maven_archives ( + archive_id INTEGER NOT NULL PRIMARY KEY REFERENCES archiveinfo(id), + group_id TEXT NOT NULL, + artifact_id TEXT NOT NULL, + version TEXT NOT NULL +) WITHOUT OIDS; + +CREATE TABLE image_archives ( + archive_id INTEGER NOT NULL PRIMARY KEY REFERENCES archiveinfo(id), + arch VARCHAR(16) NOT NULL +) WITHOUT OIDS; + +-- tracks the rpm contents of an image or other archive +CREATE TABLE archive_rpm_components ( + archive_id INTEGER NOT NULL REFERENCES archiveinfo(id), + rpm_id INTEGER NOT NULL REFERENCES rpminfo(id), + UNIQUE (archive_id, rpm_id) +) WITHOUT OIDS; +CREATE INDEX rpm_components_idx on archive_rpm_components(rpm_id); + +-- track the archive contents of an image or other archive +CREATE TABLE archive_components ( + archive_id INTEGER NOT NULL REFERENCES archiveinfo(id), + component_id INTEGER NOT NULL REFERENCES archiveinfo(id), + UNIQUE (archive_id, component_id) +) WITHOUT OIDS; +CREATE INDEX archive_components_idx on archive_components(component_id); + + +CREATE TABLE buildroot_archives ( + buildroot_id INTEGER NOT NULL REFERENCES buildroot (id), + archive_id INTEGER NOT NULL REFERENCES archiveinfo (id), + project_dep BOOLEAN NOT NULL, + PRIMARY KEY (buildroot_id, archive_id) +) WITHOUT OIDS; +CREATE INDEX buildroot_archives_archive_idx ON buildroot_archives (archive_id); + +-- Extended information about files built in Windows VMs +CREATE TABLE win_archives ( + archive_id INTEGER NOT NULL PRIMARY KEY REFERENCES archiveinfo(id), + relpath TEXT NOT NULL, + platforms TEXT NOT NULL, + flags TEXT +) WITHOUT OIDS; + + +-- Message queue for the protonmsg plugin +CREATE TABLE proton_queue ( + id SERIAL PRIMARY KEY, + created_ts TIMESTAMPTZ DEFAULT NOW(), + address TEXT NOT NULL, + props JSON NOT NULL, + body JSON NOT NULL +) WITHOUT OIDS; + + +COMMIT WORK; From 6ae3150d04767c71cfbf4dc787da7b775a5a9945 Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Sep 10 2021 01:12:04 +0000 Subject: [PATCH 5/11] koji-hub import sql schema --- diff --git a/operator/roles/koji-hub/tasks/main.yml b/operator/roles/koji-hub/tasks/main.yml index d4875de..1a83872 100644 --- a/operator/roles/koji-hub/tasks/main.yml +++ b/operator/roles/koji-hub/tasks/main.yml @@ -188,17 +188,28 @@ login_host: "{{ psql_host }}" login_user: "{{ psql_secret.data.POSTGRES_USER | b64decode }}" login_password: "{{ psql_secret.data.POSTGRES_PASSWORD | b64decode }}" - query: INSERT INTO users(name, status, usertype) VALUES('{{ koji_hub_admin_username }}', 0, 0) ON CONFLICT DO NOTHING + path_to_script: /usr/share/doc/koji/docs/schema.sql + positional_args: + - 1 + failed_when: false + # register: psql_schema + # failed_when: "not 'already exists' in psql_schema.statusmessage" - postgresql_query: db: "{{ psql_secret.data.POSTGRES_DB | b64decode }}" login_host: "{{ psql_host }}" login_user: "{{ psql_secret.data.POSTGRES_USER | b64decode }}" login_password: "{{ psql_secret.data.POSTGRES_PASSWORD | b64decode }}" - query: SELECT * FROM users WHERE name='{{ koji_hub_admin_username }}' + query: "INSERT INTO users(name, status, usertype) VALUES('{{ koji_hub_admin_username }}', 0, 0) ON CONFLICT DO NOTHING" + - postgresql_query: + db: "{{ psql_secret.data.POSTGRES_DB | b64decode }}" + login_host: "{{ psql_host }}" + login_user: "{{ psql_secret.data.POSTGRES_USER | b64decode }}" + login_password: "{{ psql_secret.data.POSTGRES_PASSWORD | b64decode }}" + query: "SELECT * FROM users WHERE name='{{ koji_hub_admin_username }}'" register: psql_user_query - postgresql_query: db: "{{ psql_secret.data.POSTGRES_DB | b64decode }}" login_host: "{{ psql_host }}" login_user: "{{ psql_secret.data.POSTGRES_USER | b64decode }}" login_password: "{{ psql_secret.data.POSTGRES_PASSWORD | b64decode }}" - query: INSERT INTO user_perms (user_id, perm_id, creator_id) VALUES ({{ psql_user_query.query_result[0].id }}, 1, 1) ON CONFLICT DO NOTHING + query: "INSERT INTO user_perms (user_id, perm_id, creator_id) VALUES ({{ psql_user_query.query_result[0].id }}, 1, 1) ON CONFLICT DO NOTHING" From 76643079062f50041c5ae1af77d4f5dbe8a2879e Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Sep 10 2021 01:13:58 +0000 Subject: [PATCH 6/11] change to fedora:34 for all components --- diff --git a/operator/config/samples/buildsys_v1alpha1_kojibuilder.yaml b/operator/config/samples/buildsys_v1alpha1_kojibuilder.yaml index 884a65e..6ec4472 100644 --- a/operator/config/samples/buildsys_v1alpha1_kojibuilder.yaml +++ b/operator/config/samples/buildsys_v1alpha1_kojibuilder.yaml @@ -3,7 +3,8 @@ kind: KojiBuilder metadata: name: sample spec: - image: quay.io/fedora/koji-builder:latest + # image: quay.io/fedora/koji-builder:latest + image: quay.io/lrossett/koji-builder:f34 replicas: 1 configmap: koji-builder-configmap cacert_secret: koji-hub-ca-cert @@ -20,4 +21,4 @@ spec: - createrepo host_name: mbbox.default ssl_verify: false - shared_pvc: koji-hub-mnt-pvc \ No newline at end of file + shared_pvc: koji-hub-mnt-pvc diff --git a/operator/config/samples/buildsys_v1alpha1_kojihub.yaml b/operator/config/samples/buildsys_v1alpha1_kojihub.yaml index 3aa7250..0e29375 100644 --- a/operator/config/samples/buildsys_v1alpha1_kojihub.yaml +++ b/operator/config/samples/buildsys_v1alpha1_kojihub.yaml @@ -3,7 +3,7 @@ kind: KojiHub metadata: name: sample spec: - image: quay.io/fedora/koji-hub:latest + image: quay.io/lrossett/koji-hub:f34 replicas: 1 persistent: true host: koji.mbox.dev # change it to match the external web url/route of koji-hub @@ -25,4 +25,4 @@ spec: web_client_cert_secret: koji-hub-web-client-cert web_client_username: kojiweb admin_client_cert: koji-hub-admin-cert - admin_username: kojiadmin \ No newline at end of file + admin_username: kojiadmin diff --git a/operator/config/samples/buildsys_v1alpha1_kojira.yaml b/operator/config/samples/buildsys_v1alpha1_kojira.yaml index 3b54b5c..07c3c8e 100644 --- a/operator/config/samples/buildsys_v1alpha1_kojira.yaml +++ b/operator/config/samples/buildsys_v1alpha1_kojira.yaml @@ -4,7 +4,8 @@ metadata: name: sample spec: replicas: 1 - image: quay.io/fedora/kojira:latest + # image: quay.io/fedora/kojira:latest + image: quay.io/lrossett/kojira:f34 configmap: kojira-config hub_username: kojira hub_host: koji-hub:8443 @@ -14,4 +15,4 @@ spec: cacert_secret: koji-hub-ca-cert client_cert_secret: kojira-client-cert shared_pvc: koji-hub-mnt-pvc - admin_secret: koji-hub-admin-cert \ No newline at end of file + admin_secret: koji-hub-admin-cert diff --git a/operator/molecule/default/verify.yml b/operator/molecule/default/verify.yml index 6decf47..dcf6418 100644 --- a/operator/molecule/default/verify.yml +++ b/operator/molecule/default/verify.yml @@ -15,9 +15,9 @@ include_tasks: 'tasks/{{ item }}_test.yml' with_items: - kojihub - - kojibuilder - - kojira - - kojiuser + # - kojibuilder + # - kojira + # - kojiuser rescue: - name: Retrieve relevant resources k8s_info: From d968b64372b06bab8ebe21064b7ce2aff18e80eb Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Sep 10 2021 01:14:10 +0000 Subject: [PATCH 7/11] using f34 tags in sample crs --- diff --git a/operator/config/samples/buildsys_v1alpha1_kojibuilder.yaml b/operator/config/samples/buildsys_v1alpha1_kojibuilder.yaml index 6ec4472..e0b8a37 100644 --- a/operator/config/samples/buildsys_v1alpha1_kojibuilder.yaml +++ b/operator/config/samples/buildsys_v1alpha1_kojibuilder.yaml @@ -3,8 +3,7 @@ kind: KojiBuilder metadata: name: sample spec: - # image: quay.io/fedora/koji-builder:latest - image: quay.io/lrossett/koji-builder:f34 + image: quay.io/fedora/koji-builder:f34 replicas: 1 configmap: koji-builder-configmap cacert_secret: koji-hub-ca-cert diff --git a/operator/config/samples/buildsys_v1alpha1_kojihub.yaml b/operator/config/samples/buildsys_v1alpha1_kojihub.yaml index 0e29375..79bb69e 100644 --- a/operator/config/samples/buildsys_v1alpha1_kojihub.yaml +++ b/operator/config/samples/buildsys_v1alpha1_kojihub.yaml @@ -3,7 +3,7 @@ kind: KojiHub metadata: name: sample spec: - image: quay.io/lrossett/koji-hub:f34 + image: quay.io/fedora/koji-hub:f34 replicas: 1 persistent: true host: koji.mbox.dev # change it to match the external web url/route of koji-hub diff --git a/operator/config/samples/buildsys_v1alpha1_kojira.yaml b/operator/config/samples/buildsys_v1alpha1_kojira.yaml index 07c3c8e..545b2f8 100644 --- a/operator/config/samples/buildsys_v1alpha1_kojira.yaml +++ b/operator/config/samples/buildsys_v1alpha1_kojira.yaml @@ -4,8 +4,7 @@ metadata: name: sample spec: replicas: 1 - # image: quay.io/fedora/kojira:latest - image: quay.io/lrossett/kojira:f34 + image: quay.io/fedora/kojira:f34 configmap: kojira-config hub_username: kojira hub_host: koji-hub:8443 diff --git a/operator/config/samples/buildsys_v1alpha1_kojiuser.yaml b/operator/config/samples/buildsys_v1alpha1_kojiuser.yaml index 9a63423..4f07063 100644 --- a/operator/config/samples/buildsys_v1alpha1_kojiuser.yaml +++ b/operator/config/samples/buildsys_v1alpha1_kojiuser.yaml @@ -10,4 +10,4 @@ spec: authentication: ssl: client_secret_name: koji-sample-user-client-cert - ca_secret_name: koji-hub-ca-cert \ No newline at end of file + ca_secret_name: koji-hub-ca-cert From 844cefc49c90b3b66ddca73bf32e940dbcaabd12 Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Sep 10 2021 01:14:10 +0000 Subject: [PATCH 8/11] koji-hub image fix and static files --- diff --git a/images/koji-hub/Dockerfile b/images/koji-hub/Dockerfile index 64cec29..3cd3bfe 100644 --- a/images/koji-hub/Dockerfile +++ b/images/koji-hub/Dockerfile @@ -1,10 +1,9 @@ FROM fedora:34 -RUN yum install -y \ +RUN dnf install --setopt=tsflags= -y \ koji koji-hub koji-web koji-hub-plugins \ curl httpd mod_ssl postgresql fedora-messaging python3-mod_wsgi -RUN curl https://pagure.io/fedora-infra/ansible/raw/master/f/roles/koji_hub/templates/fedmsg-koji-plugin.py -o /usr/lib/koji-hub-plugins/fedmsg-koji-plugin.py RUN rm -f /etc/kojiweb/web.conf && \ ln -s /etc/koji-hub/kojiweb.conf /etc/kojiweb/web.conf RUN chown -R 1001:root /etc/koji-hub && \ @@ -12,11 +11,12 @@ chown -R 1001:root /etc/pki/tls RUN mkdir -p /var/cache/kojihub && \ chown -R 1001:root /var/cache/kojihub -COPY readiness.sh /readiness.sh -COPY entrypoint.sh /entrypoint.sh +COPY files/readiness.sh /readiness.sh +COPY files/entrypoint.sh /entrypoint.sh +COPY files/fedmsg-koji-plugin.py /usr/lib/koji-hub-plugins/fedmsg-koji-plugin.py RUN chmod +x /entrypoint.sh /readiness.sh && \ -chown 1001:root /entrypoint.sh /readiness.sh +chown 1001:root /entrypoint.sh /readiness.sh /usr/lib/koji-hub-plugins/fedmsg-koji-plugin.py ENV USER=1001 diff --git a/images/koji-hub/entrypoint.sh b/images/koji-hub/entrypoint.sh deleted file mode 100755 index a181476..0000000 --- a/images/koji-hub/entrypoint.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -if [ "${KOJI_HUB_CA_CERT_PATH}" == "" ]; then - KOJI_HUB_CA_CERT_PATH="/etc/cacert/cert" -fi - -ln -s ${KOJI_HUB_CA_CERT_PATH} /etc/pki/tls/certs/`openssl x509 -hash -noout -in ${KOJI_HUB_CA_CERT_PATH}`.0 -update-ca-trust - -if [ ! -f "/var/cache/kojihub/.dbschema" ]; then - PGPASSWORD="${POSTGRES_PASSWORD}" psql \ - -h ${POSTGRES_HOST} \ - -U ${POSTGRES_USER} \ - -W ${POSTGRES_DB} < /usr/share/doc/koji/docs/schema.sql - - touch /var/cache/kojihub/.dbschema -fi - -mkdir -p /httpdir/run/ -ln -s /usr/lib64/httpd/modules /httpdir/modules -truncate --size=0 /httpdir/accesslog /httpdir/errorlog -tail -qf /httpdir/accesslog /httpdir/errorlog & -ulimit -c 0 -mkdir -p /mnt/koji/{packages,repos,work,scratch,repos-dist} -exec httpd -f /etc/koji-hub/httpd.conf -DFOREGROUND -DNO_DETACH diff --git a/images/koji-hub/files/entrypoint.sh b/images/koji-hub/files/entrypoint.sh new file mode 100755 index 0000000..a181476 --- /dev/null +++ b/images/koji-hub/files/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +if [ "${KOJI_HUB_CA_CERT_PATH}" == "" ]; then + KOJI_HUB_CA_CERT_PATH="/etc/cacert/cert" +fi + +ln -s ${KOJI_HUB_CA_CERT_PATH} /etc/pki/tls/certs/`openssl x509 -hash -noout -in ${KOJI_HUB_CA_CERT_PATH}`.0 +update-ca-trust + +if [ ! -f "/var/cache/kojihub/.dbschema" ]; then + PGPASSWORD="${POSTGRES_PASSWORD}" psql \ + -h ${POSTGRES_HOST} \ + -U ${POSTGRES_USER} \ + -W ${POSTGRES_DB} < /usr/share/doc/koji/docs/schema.sql + + touch /var/cache/kojihub/.dbschema +fi + +mkdir -p /httpdir/run/ +ln -s /usr/lib64/httpd/modules /httpdir/modules +truncate --size=0 /httpdir/accesslog /httpdir/errorlog +tail -qf /httpdir/accesslog /httpdir/errorlog & +ulimit -c 0 +mkdir -p /mnt/koji/{packages,repos,work,scratch,repos-dist} +exec httpd -f /etc/koji-hub/httpd.conf -DFOREGROUND -DNO_DETACH diff --git a/images/koji-hub/files/fedmsg-koji-plugin.py b/images/koji-hub/files/fedmsg-koji-plugin.py new file mode 100644 index 0000000..69e01f9 --- /dev/null +++ b/images/koji-hub/files/fedmsg-koji-plugin.py @@ -0,0 +1,257 @@ +# Koji callback for sending notifications about events to the fedmsg message bus +# Copyright (c) 2009-2019 Red Hat, Inc. +# +# Source: https://pagure.io/koji-fedmsg-plugin/ +# +# Authors: +# Ralph Bean +# Mike Bonnet + +import logging +import re +import time + +from koji.context import context +from koji.plugin import callbacks +from koji.plugin import callback +from koji.plugin import ignore_error +import fedora_messaging.api +import fedora_messaging.exceptions +import kojihub + + +MAX_KEY_LENGTH = 255 +log = logging.getLogger(__name__) + + +def camel_to_dots(name): + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1.\2', name) + return re.sub('([a-z0-9])([A-Z])', r'\1.\2', s1).lower() + + +def serialize_datetime_in_task(task): + date_fields = [ + "completion_time", "create_time", "start_time", "buildtime", + "creation_ts", "creation_time", + ] + for date_key in date_fields: + if task.get(date_key) is None: + continue + if isinstance(task[date_key], (float, int)): + continue + task[date_key] = time.mktime(task[date_key].timetuple()) + + +def get_message_body(topic, *args, **kws): + msg = {} + + if topic == 'package.list.change': + msg['tag'] = kws['tag']['name'] + msg['package'] = kws['package']['name'] + msg['action'] = kws['action'] + if 'owner' in kws: + msg['owner'] = kojihub.get_user(kws['owner'])['name'] + else: + msg['owner'] = None + msg['block'] = kws.get('block', None) + msg['extra_arches'] = kws.get('extra_arches', None) + msg['force'] = kws.get('force', None) + msg['update'] = kws.get('update', None) + elif topic == 'task.state.change': + info = kws['info'] + serialize_datetime_in_task(info) + + # Stuff in information about descendant tasks + task = kojihub.Task(info['id']) + info['children'] = [] + for child_orig in task.getChildren(): + child = child_orig.copy() + serialize_datetime_in_task(child) + info['children'].append(child) + + # Send the whole info dict along because it might have useful info. + # For instance, it contains the mention of what format createAppliance + # is using (raw or qcow2). + msg['info'] = info + msg['method'] = kws['info']['method'] + msg['attribute'] = kws['attribute'] + msg['old'] = kws['old'] + msg['new'] = kws['new'] + msg['id'] = kws['info']['id'] + + # extract a useful identifier from the request string + request = kws.get('info', {}).get('request', ['/']) + msg['srpm'] = request[0].split('/')[-1] + + if 'owner_name' in info: + msg['owner'] = info['owner_name'] + elif 'owner_id' in info: + msg['owner'] = kojihub.get_user(info['owner_id'])['name'] + elif 'owner' in info: + msg['owner'] = kojihub.get_user(info['owner'])['name'] + else: + msg['owner'] = None + + elif topic == 'build.state.change': + info = kws['info'] + msg['name'] = info['name'] + msg['version'] = info['version'] + msg['release'] = info['release'] + msg['epoch'] = info.get('epoch') + msg['attribute'] = kws['attribute'] + msg['old'] = kws['old'] + msg['new'] = kws['new'] + msg['build_id'] = info.get('id', None) + msg['task_id'] = info.get('task_id', None) + + if msg['task_id']: + task = kojihub.Task(msg['task_id']) + msg['request'] = task.getRequest() + else: + msg['request'] = None + + if 'owner_name' in info: + msg['owner'] = info['owner_name'] + elif 'owner_id' in info: + msg['owner'] = kojihub.get_user(info['owner_id'])['name'] + elif 'owner' in info: + msg['owner'] = kojihub.get_user(info['owner'])['name'] + else: + msg['owner'] = None + + elif topic == 'import': + # TODO -- import is currently unused. + # Should we remove it? + msg['type'] = kws['type'] + elif topic in ('tag', 'untag'): + msg['tag'] = kws['tag']['name'] + build = kws['build'] + msg['name'] = build['name'] + msg['version'] = build['version'] + msg['release'] = build['release'] + msg['user'] = kws['user']['name'] + msg['owner'] = kojihub.get_user(kws['build']['owner_id'])['name'] + msg['tag_id'] = kws['tag']['id'] + msg['build_id'] = kws['build']['id'] + elif topic == 'repo.init': + msg['tag'] = kws['tag']['name'] + msg['tag_id'] = kws['tag']['id'] + msg['repo_id'] = kws['repo_id'] + elif topic == 'repo.done': + msg['tag'] = kws['repo']['tag_name'] + msg['tag_id'] = kws['repo']['tag_id'] + msg['repo_id'] = kws['repo']['id'] + elif topic == 'rpm.sign': + if 'attribute' in kws: + # v1.10.1 and earlier + msg['attribute'] = kws['attribute'] + msg['old'] = kws['old'] + msg['new'] = kws['new'] + msg['info'] = kws['info'] + else: + # v1.11.0 (and maybe higher, but who knows) + msg['sigkey'] = kws['sigkey'] + msg['sighash'] = kws['sighash'] + msg['build'] = kws['build'] + msg['rpm'] = kws['rpm'] + serialize_datetime_in_task(msg['build']) + serialize_datetime_in_task(msg['rpm']) + + return msg + + +# This callback gets run for every koji event that starts with "post" +@callback(*[ + c for c in list(callbacks.keys()) + if c.startswith('post') and c not in [ + 'postImport', # This is kind of useless; also noisy. + # This one is special, and is called every time, so ignore it. + # Added here https://pagure.io/koji/pull-request/148 + 'postCommit', + ] +]) +@ignore_error +def queue_message(cbtype, *args, **kws): + if cbtype.startswith('post'): + msgtype = cbtype[4:] + else: + msgtype = cbtype[3:] + + # Short-circuit ourselves for task events. They are very spammy and we are + # only interested in state changes to scratch builds (parent tasks). + if cbtype == 'postTaskStateChange': + # only state changes + if not kws.get('attribute', None) == 'state': + return + # only parent tasks + if kws.get('info', {}).get('parent'): + return + # only scratch builds + request = kws.get('info', {}).get('request', [{}])[-1] + if not isinstance(request, dict) or not request.get('scratch'): + return + + topic = camel_to_dots(msgtype) + body = get_message_body(topic, *args, **kws) + + # We need this to distinguish between messages from primary koji + # and the secondary hubs off for s390 and ppc. + body['instance'] = 'primary' + + # Don't publish these uninformative rpm.sign messages if there's no actual + # sigkey present. Koji apparently adds a dummy sig value when rpms are + # first imported and there's no need to spam the world about that. + if topic == 'rpm.sign' and (body.get('info', {}).get('sigkey') == '' or + body.get('sigkey') == ''): + return + + # Also, do not want to send a message on volume_id changes + if topic == 'build.state.change' and body.get('attribute') == 'volume_id': + return + + # Last thing to do before publishing: scrub some problematic fields + # These fields are floating points which get json-encoded differently on + # rhel and fedora. + problem_fields = ['weight', 'start_ts', 'create_ts', 'completion_ts'] + + def scrub(obj): + if isinstance(obj, list): + return [scrub(item) for item in obj] + if isinstance(obj, dict): + return dict([ + (k, scrub(v)) + for k, v in list(obj.items()) + if k not in problem_fields + ]) + return obj + + body = scrub(body) + + # Queue the message for later. + # It will only get sent after postCommit is called. + messages = getattr(context, 'fedmsg_plugin_messages', []) + messages.append(dict(topic=topic, msg=body)) + context.fedmsg_plugin_messages = messages + + +# Meanwhile, postCommit actually sends messages. +@callback('postCommit') +@ignore_error +def send_messages(cbtype, *args, **kws): + messages = getattr(context, 'fedmsg_plugin_messages', []) + + for message in messages: + try: + msg = fedora_messaging.api.Message( + topic="buildsys.{}".format(message['topic']), + body=message['msg'] + ) + fedora_messaging.api.publish(msg) + except fedora_messaging.exceptions.PublishReturned as e: + log.warning( + "Fedora Messaging broker rejected message %s: %s", msg.id, e + ) + except fedora_messaging.exceptions.ConnectionException as e: + log.warning("Error sending message %s: %s", msg.id, e) + except Exception: + log.exception("Un-expected error sending fedora-messaging message") diff --git a/images/koji-hub/files/readiness.sh b/images/koji-hub/files/readiness.sh new file mode 100755 index 0000000..612db96 --- /dev/null +++ b/images/koji-hub/files/readiness.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +BODY='system.listMethods' + +res=$(curl \ +-X POST \ +-H 'Content-Type: text/xml' \ +-d "$BODY" \ +http://127.0.0.1:8080/kojihub/) + +if [ "$?" != '0' ] +then + exit 1 +fi + +if [ "$res" == '**' ] +then + echo $res + exit 1 +fi + +echo $http_code + +exit 0 \ No newline at end of file diff --git a/images/koji-hub/readiness.sh b/images/koji-hub/readiness.sh deleted file mode 100755 index 612db96..0000000 --- a/images/koji-hub/readiness.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -BODY='system.listMethods' - -res=$(curl \ --X POST \ --H 'Content-Type: text/xml' \ --d "$BODY" \ -http://127.0.0.1:8080/kojihub/) - -if [ "$?" != '0' ] -then - exit 1 -fi - -if [ "$res" == '**' ] -then - echo $res - exit 1 -fi - -echo $http_code - -exit 0 \ No newline at end of file diff --git a/images/koji-hub/service.yaml b/images/koji-hub/service.yaml deleted file mode 100644 index 3fbc135..0000000 --- a/images/koji-hub/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: postgresql - labels: - app: postgres -spec: - type: NodePort - ports: - - port: 5432 - selector: - app: postgres From 82aa41e23c1f3b1f519cc8d60e3521c0d9d48d0d Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Sep 10 2021 01:14:10 +0000 Subject: [PATCH 9/11] using correct variable for the error message --- diff --git a/operator/roles/koji-lib/library/koji_user.py b/operator/roles/koji-lib/library/koji_user.py index 2387dca..aff46e5 100644 --- a/operator/roles/koji-lib/library/koji_user.py +++ b/operator/roles/koji-lib/library/koji_user.py @@ -117,7 +117,7 @@ def ssl_config(module): module.fail_json(changed=False, skipped=False, failed=True, - error='Missing ssl_auth "%s" key.' % e.args[0]) + msg='Missing ssl_auth "%s" key.' % e.args[0]) def main(): @@ -130,7 +130,7 @@ def main(): module.fail_json(changed=False, skipped=False, failed=True, - error='Missing authentication config') + msg='Missing authentication config') options = Values(config) session_opts = koji.grab_session_options(options) @@ -142,20 +142,20 @@ def main(): module.fail_json(changed=False, skipped=False, failed=True, - error=str(e)) + msg=str(e)) username = module.params['username'] perms = module.params['permissions'] user = session.getUser(username) - if not user: + if not user: try: user = session.createUser(username) except Exception as e: module.fail_json(changed=False, skipped=False, failed=True, - error=str(e)) + msg=str(e)) for perm in perms: try: @@ -174,4 +174,4 @@ def main(): if __name__ == '__main__': - main() \ No newline at end of file + main() From adeedbd82df84e2f599c90e54ce615922d0f8d08 Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Sep 10 2021 01:14:10 +0000 Subject: [PATCH 10/11] removing unused image files --- diff --git a/images/identity/Dockerfile b/images/identity/Dockerfile deleted file mode 100644 index 4508f61..0000000 --- a/images/identity/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM fedora:31 - -ENV PSQL_USER=mbox \ -PSQL_PASS=mbox \ -PSQL_HOST=postgre \ -PSQL_DB=mbox-identity - -RUN dnf install -y ipsilon ipsilon-openidc python2-psycopg2 curl httpd mod_ssl python3-mod_wsgi - -COPY start.sh /etc/ipsilon-cfg/start.sh -RUN chmod +x /etc/ipsilon-cfg/start.sh && \ -chown -R 1001:root /etc/ipsilon-cfg -RUN printf "def fix_user_dirs(*args, **kwargs):\n pass\n" >>/usr/lib/python3.7/site-packages/ipsilon/tools/files.py - -COPY cfgprofile /etc/ipsilon-cfg/cfgprofile -RUN mkdir /etc/ipsilon/{httpd,ipsilon,data} && \ -LOGFILE=/dev/stdout ipsilon-server-install --config-profile=/etc/ipsilon-cfg/cfgprofile - -EXPOSE 8443 - -ENTRYPOINT /etc/ipsilon-cfg/start.sh \ No newline at end of file diff --git a/images/identity/cfgprofile b/images/identity/cfgprofile deleted file mode 100644 index 51f9766..0000000 --- a/images/identity/cfgprofile +++ /dev/null @@ -1,18 +0,0 @@ -[globals] -datadir = /etc/ipsilon/data -httpdconfd = /etc/ipsilon/httpd -confdir = /etc/ipsilon/ipsilon - -[arguments] -ignored_database_url = postgresql://%(PSQL_USER):%(PSQL_PASS)@%(PSQL_HOST)/%(PSQL_DB) -admin_user = admin -gssapi = no -ipa = no -port = 8443 -testauth = yes -openidc = yes -hostname = identity.mbox.test -server_debugging = True -root_instance = yes -instance = root -pam = no \ No newline at end of file diff --git a/images/identity/start.sh b/images/identity/start.sh deleted file mode 100755 index f2a81dd..0000000 --- a/images/identity/start.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -xe - -mkdir /httpdir/run || rm -rf /httpdir/run/* -ln -s /etc/httpd/modules /httpdir/modules -truncate --size=0 /httpdir/accesslog /httpdir/errorlog -ulimit -c 0 - -exec httpd -f /etc/ipsilon-cfg/httpd/httpd.conf -DFOREGROUND -DNO_DETACH \ No newline at end of file diff --git a/images/kerberos/Dockerfile b/images/kerberos/Dockerfile deleted file mode 100644 index 83234d1..0000000 --- a/images/kerberos/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM fedora:33 - -RUN dnf install -y krb5-libs krb5-server krb5-workstation expect - -EXPOSE 88 -EXPOSE 749 \ No newline at end of file diff --git a/images/mbs-backend/Dockerfile b/images/mbs-backend/Dockerfile deleted file mode 100644 index c42c9f7..0000000 --- a/images/mbs-backend/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM fedora:33 - -RUN yum install -y module-build-service python3-psycopg2 python3-fedmsg - -RUN mkdir /etc/mbs-backend && \ -chown -R 1001:root /etc/mbs-backend - -COPY entrypoint.sh /entrypoint.sh -COPY mbs-scheduler.py /etc/mbs-backend/mbs-scheduler.py - -ENV USER=1001 - -ENTRYPOINT /entrypoint.sh diff --git a/images/mbs-backend/entrypoint.sh b/images/mbs-backend/entrypoint.sh deleted file mode 100755 index 4519cdd..0000000 --- a/images/mbs-backend/entrypoint.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Start the fedmsg-hub -echo "Starting fedmsg-hub" -exec /usr/bin/fedmsg-hub-3 - diff --git a/images/mbs-backend/mbs-scheduler.py b/images/mbs-backend/mbs-scheduler.py deleted file mode 100644 index 915506b..0000000 --- a/images/mbs-backend/mbs-scheduler.py +++ /dev/null @@ -1,10 +0,0 @@ -import os - -def str2bool(v): - return v.lower() in ['true', 't', '1', 'yes', 'y'] - - -config = { - 'mbsconsumer': str2bool(os.environ.get('MBS_CONSUMER', 'true')), - 'mbspoller': str2bool(os.environ.get('MBS_POLLER', 'true')), -} diff --git a/images/mbs-frontend/Dockerfile b/images/mbs-frontend/Dockerfile deleted file mode 100644 index 3f3509d..0000000 --- a/images/mbs-frontend/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM fedora:33 - -RUN yum install -y module-build-service python3-psycopg2 httpd python3-mod_wsgi mod_ssl - -COPY entrypoint.sh /entrypoint.sh -COPY mbs.wsgi /mbs.wsgi -RUN chmod +x /entrypoint.sh - -ENV USER=1001 -EXPOSE 8443 - -ENTRYPOINT /entrypoint.sh diff --git a/images/mbs-frontend/entrypoint.sh b/images/mbs-frontend/entrypoint.sh deleted file mode 100644 index b70d4dd..0000000 --- a/images/mbs-frontend/entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/bash - -update-ca-trust -mkdir -p /httpdir/run -ln -s /usr/lib64/httpd/modules /httpdir/modules -truncate --size=0 /httpdir/accesslog /httpdir/errorlog -tail -qf /httpdir/accesslog /httpdir/errorlog & -ulimit -c 0 -exec httpd -f /etc/mbs-frontend/httpd.conf -DFOREGROUND -DNO_DETACH diff --git a/images/mbs-frontend/mbs.wsgi b/images/mbs-frontend/mbs.wsgi deleted file mode 100644 index f8dfe9b..0000000 --- a/images/mbs-frontend/mbs.wsgi +++ /dev/null @@ -1,3 +0,0 @@ -import logging -logging.basicConfig(level=logging.DEBUG) -from module_build_service import app as application \ No newline at end of file From f566abda4d895eb46a845ff4ba41e83cd8e166b6 Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Sep 10 2021 09:49:35 +0000 Subject: [PATCH 11/11] verify playbook changes --- diff --git a/operator/molecule/default/verify.yml b/operator/molecule/default/verify.yml index dcf6418..6decf47 100644 --- a/operator/molecule/default/verify.yml +++ b/operator/molecule/default/verify.yml @@ -15,9 +15,9 @@ include_tasks: 'tasks/{{ item }}_test.yml' with_items: - kojihub - # - kojibuilder - # - kojira - # - kojiuser + - kojibuilder + - kojira + - kojiuser rescue: - name: Retrieve relevant resources k8s_info: