From 941e4dc01b1c8353332d0aa5ba32d7304224797b Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Aug 02 2019 05:08:42 +0000 Subject: [PATCH 1/3] Add devel documentation about container images rebuild. --- diff --git a/docs/conf.py b/docs/conf.py index b6669c2..7c4901e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,7 +52,7 @@ master_doc = 'index' # General information about the project. project = 'Freshmaker' -copyright = '2017, Red Hat, Inc. and others' +copyright = '2019, Red Hat, Inc. and others' author = 'Red Hat, Inc. and others' # The version info for the project you're documenting, acts as replacement for diff --git a/docs/images_rebuild.rst b/docs/images_rebuild.rst new file mode 100644 index 0000000..5f5311a --- /dev/null +++ b/docs/images_rebuild.rst @@ -0,0 +1,118 @@ +=========================================== +Rebuilding container images affected by CVE +=========================================== + +This document is intended for Freshmaker developers to understand the general algorithm used by Freshmaker to find out the container images which will be rebuild as result of release of RPM fixing the CVE. + +It describes the ``Lightblue.find_images_to_rebuild`` method from high-level perspective. + + +Images stored in Lightblue database +=================================== + +There are two main objects in the Lightblue which are interesting for Freshmaker: + +- ``ContainerImage`` - Stores metadata about particular container image build. +- ``ContainerRepository`` - Groups multiple images together and also contains metadata about the container repository itself. + +Important facts about ``ContainerImage``: + +- There is one ``ContainerImage`` record for each architecture on which the image has been built. +- If the image is published, it is "squashed", so it is not possible to find out its direct parent once it is published. +- If the image is published, extra ``ContainerImage`` record is created with ``published: True`` in metadata. + +Important facts about ``ContainerRepository``: + +- The ``ContainerImage`` records are tagged in ``ContainerRepository`` with tags. By default, latest built image has ``latest`` tag, but there might be multiple tags, for example per container images major release. +- The ``ContainerRepository`` contains ``auto_rebuild_tags`` list which defines which tags within the ``ContainerRepository`` are enabled to be handled by Freshmaker. + + +Resolving container images +========================== + +The metadata stored in ``ContainerImage`` in Lightblue are not enough to rebuild the container image. + +Freshmaker for example need following additional metadata, for example: + +- The git repository, branch name and commit hash from which the image have been built, so we can resubmit the exactl same commit. +- The name of Koji target in which the image has been built. +- The list of architectures the image has been built for. + +The process of getting these data is called ``resolving the image`` and happens in ``ContainerImage.resolve`` method. + + +Finding published affected images +================================= + +The ``Lightblue.find_images_to_rebuild`` has two important input values: + +- ``srpm_nvrs`` - The list of NVRs of SRPMs from which the packages fixing the CVE(s) are built. +- ``content_sets`` - The list of content sets (basically pointers to particular products) in which the RPMs are released. + +The main goal is to find out all the container images which **contains some RPM comming from srpm_nvrs with older version and was installing this RPM from one of the content_sets**. + +Freshmaker only cares about published container images, so at first Freshmaker needs to find out all the published container images affected by the CVE. This is done in ``Lightblue.find_images_with_packages_from_content_set`` method. + +At first, this method finds all the available container repositories which are enabled to be handled by Freshmaker (``Lightblue.find_all_container_repositories``). + +With that knowledge, it generates the Lightblue query (``Lightblue.find_images_with_included_srpms``) to find images which: + +- includes some RPM comming from SRPM from ``srpm_nvrs``. +- has one of the ``content_sets`` enabled. +- is tagged by one of the ``auto_rebuild_tags`` in one of the allowed container repositories. + +The images returned by Lightblue are ``resolved`` and goes to next step. Let's call this list of images ``published affected images``. + + +Finding unpublished affected images +=================================== + +Every image in ``published affected images`` squashed in a way that we cannot find its direct parent. Freshmaker therefore needs to find out unpublished versions of those images. Let's call this list ``unpublished affected images``. + +This is done in multiple threads which call ``Lightblue.find_unpublished_image_for_build`` method. + + +Finding affected parent images +============================== + +Freshmaker now has the list of ``unpublished affected images``. Freshmaker knows that the every image in this list is affected by the CVE, because it contains the RPM in older version than the one which fixes the CVE and this RPM comes from one of the ``content_sets``. + +The issue is, that this RPM might have been (and in most cases it is) inherited from some parent image which ``yum install`` it. + +Freshmaker therefore needs to go up in the image inheritance tree and find out the first parent image which contains this RPM. This parent image is responsible for installation of affected RPM and therefore all the parent images of particular ``unpublished affected image`` up to the first parent image which installed the RPM must be rebuilt. + +When determining the parent of the last affected parent image, Freshmaker tries to use the latest published parent. If it fails to find such image, it just uses the original parent image of this last affected parent image. + +The ``Lightblue.find_parent_images_with_package`` is called in a multiple threads for every ``unpublished affected image``. It extends the single ``unpublished affected image`` with all its affected parents which need to be rebuild. + +This data is stored in so-called ``rebuild_list`` in ``[unpublished_affected_image, affected_parent, affected_grandparent, ...]`` format. + + +Deduplicating multiple rebuild lists +==================================== + +Two ``unpublished affected images`` can easily share the same parent or grandparent image, but in different version or release. + +Freshmaker however needs to rebuild each container image in particular major version only once, otherwise it wouldn't be possible to release such images. Let's take following ``rebuild_lists`` as example: + +- ``[foo-1-1, parent-1-1]`` +- ``[bar-1-1, parent-1-3]`` + +In this case, Freshmaker can simply upgrade the ``foo-1-1`` to ``parent-1-3`` parent image and do not rebuild ``parent-1-1`` at all. + +The deduplication code respects following rules: + +- The parent image is always kept in the same ``version``. +- The parent image is updated to latest published ``release``. +- If the parent image is unpublished, it is kept in the same ``release``. +- If there are multiple child images with completely different parent images, the child images are grouped by the parent image and are updated individually. For example, if ``foo-1-1`` depends on ``bar-1-1`` and ``foo-1-2`` depend on ``parent-1-1``, then ``foo-1-1`` is not updated to ``foo-1-2`` to not break the compatibility. + +The deduplication method returns the changed ``rebuild_lists``. + + +Creating the build batches +========================== + +Freshmaker needs to know the order of images, so batches are created from the ``rebuild_lists`` in a way that the leaf images are in the last batch, their parent images in the first batch, their grandparent images in the second batch, ... + +This is the result of the ``LightBlue`` code and is handled later by other parts of ``Freshmaker``. diff --git a/docs/index.rst b/docs/index.rst index 34db17a..416b86c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ compose (mainly the RPM repository) with packages from Koji using the REST API. about api + images_rebuild Indices and tables ================== From 5246088d1fcb88e14a9697f7a464108e74c452d6 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Aug 02 2019 05:08:42 +0000 Subject: [PATCH 2/3] WIP: Update API docs. --- diff --git a/docs/api.rst b/docs/api.rst index f13077c..3474cf4 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -2,14 +2,283 @@ Freshmaker APIs =============== -.. automodule:: freshmaker +.. _event_json_api_1: + +Freshmaker Event JSON representation - API version 1 +==================================================== + +The Freshmaker Event is always represented in the API request as JSON, for example: + +.. sourcecode:: none + + { + "builds": [], + "depending_events": [], + "depends_on_events": [], + "dry_run": false, + "event_type_id": 10, + "id": 18730, + "message_id": "message_123", + "requested_rebuilds": [], + "requester": null, + "requester_metadata": {}, + "search_key": "44637", + "state": 3, + "state_name": "COMPLETE", + "state_reason": "4 images rebuilt.", + "time_created": "2019-08-01T07:00:46Z", + "time_done": "2019-08-01T07:03:12Z", + "url": "/api/1/events/18730" + } + +The fields used in the Freshmaker Event JSON have following meaning: + +.. _event_builds: + +*builds* - ``(list of Freshmaker Artifact build JSONs)`` + List of artifact builds which are part of this Freshmaker event. + +.. _event_depending_events: + +*depending_events* - ``(list of numbers)`` + List of IDs of other Freshmaker events which are depending on this Freshmaker event. + +.. _event_depends_on_events: + +*depends_on_events* - ``(list of numbers)`` + List of IDs of other Freshmaker events which this event depends on. + +.. _event_dry_run: + +*dry_run* - ``(boolean)`` + When set to ``true`` the event is handled in dry run mode. In this mode, no real builds are submitted to build system. Instead, all the builds are automatically moved to ``COMPLETE`` state. + +.. _event_event_type_id: + +*event_type_id* - ``(number)`` + The ID of the type of this Event. Full list of Event types can listed using the ``/api/1/event-types/`` REST API. The most important event types are: + + - 10 (``ErrataAdvisoryStateChangedEvent``) - Event triggered by message, rebuilding all artifacts (currently just container images) as result of advisory release. + - 13 (``ManualRebuildWithAdvisoryEvent``) - Event triggered manually, rebuilding all artifacts (currently just container images) as result of advisory release. + +.. _event_id: + +*id* - ``(number)`` + The ID of Freshmaker event. + +.. _event_message_id: + +*message_id* - ``(string)`` + The ID of message (fedmsg or AMQP) which triggered the Event. + +.. _event_requested_rebuilds: + +*requested_rebuilds* - ``(list of strings)`` + List of artifacts which should be rebuilt in this Freshmaker Event. Filled in only for manual rebuilds which requested particular artifacts to be rebuilt. Currently, it is always list of container images NVRs. + +.. _event_requester: + +*requester* - ``(string or null)`` + The Kerberos username of requester of this Freshmaker Event. Set to string only for manual rebuilds, otherwise ``null``. + +.. _event_requester_metadata: + +*requester_metadata* - ``(JSON)`` + Additional metadata set by requester when submitting the manual rebuild. It can be used to track the context of manual rebuild. + +.. _event_search_key: + +*search_key* - ``(string)`` + The key identifying source of the Event. For each Event type, there is a different way how the key is generated: + + - Event type 10 (``ErrataAdvisoryStateChangedEvent``) - The ID of advisory which triggered the Event. + - Event type 13 (``ManualRebuildWithAdvisoryEvent``) - The ID of advisory which triggered the Event. + +.. _event_state: + +*state* - ``(number)`` + Number defining the state the Event is currently in: + + - 0 (``INITIALIZED``) - Event is initialized and Freshmaker is searching for artifacts to build. + - 1 (``BUILDING``) - Some artifacts to build have been found and Freshmaker is building them. + - 2 (``COMPLETE``) - All artifacts have been build. Note that this does not mean all the builds were successfull - just one successfull artifact build is enough to mark the Event as ``COMPLETE``. + - 3 (``FAILED``) - There was some major error while handling the Event or all the artifact builds failed to build. + - 4 (``SKIPPED``) - No artifacts have been found to build or the Event is not allowed by the Freshmaker's configuration. + - 5 (``CANCELED``) - Event has been manually cancelled using the REST API. + +.. _event_state_name: + +*state_name* - ``(string)`` + Name of the state the compose is currently in. See ``state`` for more info. + +.. _event_state_reason: + +*state_reason* - ``(string)`` + Sentence describing the current state. + +.. _event_time_created: + +*_time_created* - ``(datetime)`` + The date and time on which the Event has been initialized. +.. _event_time_done: + +*time_done* - ``(datetime)`` + The date and time on which the Event has been moved to ``FAILED``, ``COMPLETE``, or ``CANCELED`` state. + + +.. _event_json_api_2: + +Freshmaker Event JSON representation - API version 2 +==================================================== + +The Freshmaker Event is always represented in the API request as JSON, for example: + +.. sourcecode:: none + + { + "builds_summary": { + "total": 0 + }, + "depending_events": [], + "depends_on_events": [], + "dry_run": false, + "event_type_id": 10, + "id": 18730, + "message_id": "message_123", + "requested_rebuilds": [], + "requester": null, + "requester_metadata": {}, + "search_key": "44637", + "state": 3, + "state_name": "COMPLETE", + "state_reason": "4 images rebuilt.", + "time_created": "2019-08-01T07:00:46Z", + "time_done": "2019-08-01T07:03:12Z", + "url": "/api/1/events/18730" + } + +There are following differences between API version 1 and API version 2: + +- The ``builds`` field is replaced with ``builds_summary``. + +.. _event_builds_summary: + +*builds_summary* - ``(JSON)`` + JSON object showing the summary of artifacts builds included in the Event grouped by their ``state_name``. For example: + + .. sourcecode:: none + + "builds_summary": { + "total": 10, + "DONE": 7, + "FAILED": 3 + } + + +.. _build_json_api_1_2: + +Artifact Build JSON representation - API version 1 and 2 +======================================================== + +The Freshmaker Artifact Build is always represented in the API request as JSON, for example: + +.. sourcecode:: none + + { + "build_args": { + ... + }, + "build_id": 22429387, + "dep_on": "fedora-30-container", + "dep_on_id": 21657, + "event_id": 16730, + "id": 21837, + "name": "httpd-container", + "odcs_composes": [], + "original_nvr": "httpd-container-2.4-1", + "rebuild_reason": "directly_affected", + "rebuilt_nvr": "httpd-container-2.4-1.1561731291", + "state": 1, + "state_name": "DONE", + "state_reason": "Built successfully.", + "time_completed": "2019-06-29T03:28:56Z", + "time_submitted": "2019-06-28T14:14:51Z", + "type": 1, + "type_name": "IMAGE", + "url": "/api/1/builds/21837" + } + +.. _build_build_args: + +*build_args* - ``(JSON)`` + JSON object containing arguments passed to build system to build this artifact. + + .. WARNING:: + The content of this JSON is not part of the Freshmaker REST API and can change at any time. + +.. _build_build_id: + +*build_id* - ``(number)`` + The ID of Artifact Build. + +.. _build_dep_on: + +*dep_on* - ``(string)`` + The :ref:`name`. of the Artifact build this Artifact build depends on. + +.. _build_dep_on_id: + +*dep_on_id* - ``(number)`` + The ID of the Artifact build this Artifact build depends on. + +.. _build_event_id: + +*event_id* - ``(number)`` + The ID of the Event this Artifact Build is part of. + +.. _build_name: + +*name* - ``(string)`` + The name of this Artifact Build. + +.. _build_odcs_composes: + +*odcs_composes* - ``(list of numbers)`` + List of ODCS composes the Freshmaker directly generated and used while building this Artifact build in the build system. + +REST API pagination +=================== + +When multiple objects are returned by the Freshmaker REST API, they are wrapped in the following JSON which allows pagination: + +.. sourcecode:: none + + { + "items": [ + {JSON_OBJECT}, + ... + ], + "meta": { + "first": "http://freshmaker.localhost/api/1/events/?per_page=10&page=1", + "last": "http://freshmaker.localhost/api/1/events/?per_page=10&page=14890", + "next": "http://freshmaker.localhost/api/1/events/?per_page=10&page=2", + "page": 1, + "pages": 14890, + "per_page": 10, + "prev": null, + "total": 148898 + } + } + +The ``items`` list contains the objects JSONs. The ``meta`` dict contains metadata about pagination. It is possible to use ``per_page`` argument to set the number of objects showed per single page and ``page`` to choose the page to show. -.. _http-api: HTTP REST API ============= +.. automodule:: freshmaker + .. autoflask:: freshmaker:app :undoc-static: :modules: freshmaker.views From fa902267e32d149ed997c988e06c2bb041c89d03 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Aug 02 2019 09:14:49 +0000 Subject: [PATCH 3/3] Describe APIv1 and APIv2 in separate docs. Describe Messaging API. --- diff --git a/docs/api.rst b/docs/api.rst deleted file mode 100644 index 3474cf4..0000000 --- a/docs/api.rst +++ /dev/null @@ -1,285 +0,0 @@ -=============== -Freshmaker APIs -=============== - -.. _event_json_api_1: - -Freshmaker Event JSON representation - API version 1 -==================================================== - -The Freshmaker Event is always represented in the API request as JSON, for example: - -.. sourcecode:: none - - { - "builds": [], - "depending_events": [], - "depends_on_events": [], - "dry_run": false, - "event_type_id": 10, - "id": 18730, - "message_id": "message_123", - "requested_rebuilds": [], - "requester": null, - "requester_metadata": {}, - "search_key": "44637", - "state": 3, - "state_name": "COMPLETE", - "state_reason": "4 images rebuilt.", - "time_created": "2019-08-01T07:00:46Z", - "time_done": "2019-08-01T07:03:12Z", - "url": "/api/1/events/18730" - } - -The fields used in the Freshmaker Event JSON have following meaning: - -.. _event_builds: - -*builds* - ``(list of Freshmaker Artifact build JSONs)`` - List of artifact builds which are part of this Freshmaker event. - -.. _event_depending_events: - -*depending_events* - ``(list of numbers)`` - List of IDs of other Freshmaker events which are depending on this Freshmaker event. - -.. _event_depends_on_events: - -*depends_on_events* - ``(list of numbers)`` - List of IDs of other Freshmaker events which this event depends on. - -.. _event_dry_run: - -*dry_run* - ``(boolean)`` - When set to ``true`` the event is handled in dry run mode. In this mode, no real builds are submitted to build system. Instead, all the builds are automatically moved to ``COMPLETE`` state. - -.. _event_event_type_id: - -*event_type_id* - ``(number)`` - The ID of the type of this Event. Full list of Event types can listed using the ``/api/1/event-types/`` REST API. The most important event types are: - - - 10 (``ErrataAdvisoryStateChangedEvent``) - Event triggered by message, rebuilding all artifacts (currently just container images) as result of advisory release. - - 13 (``ManualRebuildWithAdvisoryEvent``) - Event triggered manually, rebuilding all artifacts (currently just container images) as result of advisory release. - -.. _event_id: - -*id* - ``(number)`` - The ID of Freshmaker event. - -.. _event_message_id: - -*message_id* - ``(string)`` - The ID of message (fedmsg or AMQP) which triggered the Event. - -.. _event_requested_rebuilds: - -*requested_rebuilds* - ``(list of strings)`` - List of artifacts which should be rebuilt in this Freshmaker Event. Filled in only for manual rebuilds which requested particular artifacts to be rebuilt. Currently, it is always list of container images NVRs. - -.. _event_requester: - -*requester* - ``(string or null)`` - The Kerberos username of requester of this Freshmaker Event. Set to string only for manual rebuilds, otherwise ``null``. - -.. _event_requester_metadata: - -*requester_metadata* - ``(JSON)`` - Additional metadata set by requester when submitting the manual rebuild. It can be used to track the context of manual rebuild. - -.. _event_search_key: - -*search_key* - ``(string)`` - The key identifying source of the Event. For each Event type, there is a different way how the key is generated: - - - Event type 10 (``ErrataAdvisoryStateChangedEvent``) - The ID of advisory which triggered the Event. - - Event type 13 (``ManualRebuildWithAdvisoryEvent``) - The ID of advisory which triggered the Event. - -.. _event_state: - -*state* - ``(number)`` - Number defining the state the Event is currently in: - - - 0 (``INITIALIZED``) - Event is initialized and Freshmaker is searching for artifacts to build. - - 1 (``BUILDING``) - Some artifacts to build have been found and Freshmaker is building them. - - 2 (``COMPLETE``) - All artifacts have been build. Note that this does not mean all the builds were successfull - just one successfull artifact build is enough to mark the Event as ``COMPLETE``. - - 3 (``FAILED``) - There was some major error while handling the Event or all the artifact builds failed to build. - - 4 (``SKIPPED``) - No artifacts have been found to build or the Event is not allowed by the Freshmaker's configuration. - - 5 (``CANCELED``) - Event has been manually cancelled using the REST API. - -.. _event_state_name: - -*state_name* - ``(string)`` - Name of the state the compose is currently in. See ``state`` for more info. - -.. _event_state_reason: - -*state_reason* - ``(string)`` - Sentence describing the current state. - -.. _event_time_created: - -*_time_created* - ``(datetime)`` - The date and time on which the Event has been initialized. - -.. _event_time_done: - -*time_done* - ``(datetime)`` - The date and time on which the Event has been moved to ``FAILED``, ``COMPLETE``, or ``CANCELED`` state. - - -.. _event_json_api_2: - -Freshmaker Event JSON representation - API version 2 -==================================================== - -The Freshmaker Event is always represented in the API request as JSON, for example: - -.. sourcecode:: none - - { - "builds_summary": { - "total": 0 - }, - "depending_events": [], - "depends_on_events": [], - "dry_run": false, - "event_type_id": 10, - "id": 18730, - "message_id": "message_123", - "requested_rebuilds": [], - "requester": null, - "requester_metadata": {}, - "search_key": "44637", - "state": 3, - "state_name": "COMPLETE", - "state_reason": "4 images rebuilt.", - "time_created": "2019-08-01T07:00:46Z", - "time_done": "2019-08-01T07:03:12Z", - "url": "/api/1/events/18730" - } - -There are following differences between API version 1 and API version 2: - -- The ``builds`` field is replaced with ``builds_summary``. - -.. _event_builds_summary: - -*builds_summary* - ``(JSON)`` - JSON object showing the summary of artifacts builds included in the Event grouped by their ``state_name``. For example: - - .. sourcecode:: none - - "builds_summary": { - "total": 10, - "DONE": 7, - "FAILED": 3 - } - - -.. _build_json_api_1_2: - -Artifact Build JSON representation - API version 1 and 2 -======================================================== - -The Freshmaker Artifact Build is always represented in the API request as JSON, for example: - -.. sourcecode:: none - - { - "build_args": { - ... - }, - "build_id": 22429387, - "dep_on": "fedora-30-container", - "dep_on_id": 21657, - "event_id": 16730, - "id": 21837, - "name": "httpd-container", - "odcs_composes": [], - "original_nvr": "httpd-container-2.4-1", - "rebuild_reason": "directly_affected", - "rebuilt_nvr": "httpd-container-2.4-1.1561731291", - "state": 1, - "state_name": "DONE", - "state_reason": "Built successfully.", - "time_completed": "2019-06-29T03:28:56Z", - "time_submitted": "2019-06-28T14:14:51Z", - "type": 1, - "type_name": "IMAGE", - "url": "/api/1/builds/21837" - } - -.. _build_build_args: - -*build_args* - ``(JSON)`` - JSON object containing arguments passed to build system to build this artifact. - - .. WARNING:: - The content of this JSON is not part of the Freshmaker REST API and can change at any time. - -.. _build_build_id: - -*build_id* - ``(number)`` - The ID of Artifact Build. - -.. _build_dep_on: - -*dep_on* - ``(string)`` - The :ref:`name`. of the Artifact build this Artifact build depends on. - -.. _build_dep_on_id: - -*dep_on_id* - ``(number)`` - The ID of the Artifact build this Artifact build depends on. - -.. _build_event_id: - -*event_id* - ``(number)`` - The ID of the Event this Artifact Build is part of. - -.. _build_name: - -*name* - ``(string)`` - The name of this Artifact Build. - -.. _build_odcs_composes: - -*odcs_composes* - ``(list of numbers)`` - List of ODCS composes the Freshmaker directly generated and used while building this Artifact build in the build system. - -REST API pagination -=================== - -When multiple objects are returned by the Freshmaker REST API, they are wrapped in the following JSON which allows pagination: - -.. sourcecode:: none - - { - "items": [ - {JSON_OBJECT}, - ... - ], - "meta": { - "first": "http://freshmaker.localhost/api/1/events/?per_page=10&page=1", - "last": "http://freshmaker.localhost/api/1/events/?per_page=10&page=14890", - "next": "http://freshmaker.localhost/api/1/events/?per_page=10&page=2", - "page": 1, - "pages": 14890, - "per_page": 10, - "prev": null, - "total": 148898 - } - } - -The ``items`` list contains the objects JSONs. The ``meta`` dict contains metadata about pagination. It is possible to use ``per_page`` argument to set the number of objects showed per single page and ``page`` to choose the page to show. - - -HTTP REST API -============= - -.. automodule:: freshmaker - -.. autoflask:: freshmaker:app - :undoc-static: - :modules: freshmaker.views - :order: path diff --git a/docs/api_v1.rst b/docs/api_v1.rst new file mode 100644 index 0000000..5b793cf --- /dev/null +++ b/docs/api_v1.rst @@ -0,0 +1,304 @@ +======================== +API version 1 +======================== + +.. _event_json_api_1: + +Event JSON representation +==================================== + +The Freshmaker Event is always represented in the API request as JSON, for example: + +.. sourcecode:: none + + { + "builds": [], + "depending_events": [], + "depends_on_events": [], + "dry_run": false, + "event_type_id": 10, + "id": 18730, + "message_id": "message_123", + "requested_rebuilds": [], + "requester": null, + "requester_metadata": {}, + "search_key": "44637", + "state": 3, + "state_name": "COMPLETE", + "state_reason": "4 images rebuilt.", + "time_created": "2019-08-01T07:00:46Z", + "time_done": "2019-08-01T07:03:12Z", + "url": "/api/1/events/18730" + } + +The fields used in the Freshmaker Event JSON have following meaning: + +.. _event_builds: + +*builds* - ``(list of Freshmaker Artifact build JSONs)`` + List of artifact builds which are part of this Freshmaker event. + +.. _event_depending_events: + +*depending_events* - ``(list of numbers)`` + List of IDs of other Freshmaker events which are depending on this Freshmaker event. + +.. _event_depends_on_events: + +*depends_on_events* - ``(list of numbers)`` + List of IDs of other Freshmaker events which this event depends on. + +.. _event_dry_run: + +*dry_run* - ``(boolean)`` + When set to ``true`` the event is handled in dry run mode. In this mode, no real builds are submitted to build system. Instead, all the builds are automatically moved to ``COMPLETE`` state. + +.. _event_event_type_id: + +*event_type_id* - ``(number)`` + The ID of the type of this Event. Full list of Event types can listed using the ``/api/1/event-types/`` REST API. The most important event types are: + + - 10 (``ErrataAdvisoryStateChangedEvent``) - Event triggered by message, rebuilding all artifacts (currently just container images) as result of advisory release. + - 13 (``ManualRebuildWithAdvisoryEvent``) - Event triggered manually, rebuilding all artifacts (currently just container images) as result of advisory release. + +.. _event_id: + +*id* - ``(number)`` + The ID of Freshmaker event. + +.. _event_message_id: + +*message_id* - ``(string)`` + The ID of message (fedmsg or AMQP) which triggered the Event. + +.. _event_requested_rebuilds: + +*requested_rebuilds* - ``(list of strings)`` + List of artifacts which should be rebuilt in this Freshmaker Event. Filled in only for manual rebuilds which requested particular artifacts to be rebuilt. Currently, it is always list of container images NVRs. + +.. _event_requester: + +*requester* - ``(string or null)`` + The Kerberos username of requester of this Freshmaker Event. Set to string only for manual rebuilds, otherwise ``null``. + +.. _event_requester_metadata: + +*requester_metadata* - ``(JSON)`` + Additional metadata set by requester when submitting the manual rebuild. It can be used to track the context of manual rebuild. + +.. _event_search_key: + +*search_key* - ``(string)`` + The key identifying source of the Event. For each Event type, there is a different way how the key is generated: + + - Event type 10 (``ErrataAdvisoryStateChangedEvent``) - The ID of advisory which triggered the Event. + - Event type 13 (``ManualRebuildWithAdvisoryEvent``) - The ID of advisory which triggered the Event. + +.. _event_state: + +*state* - ``(number)`` + Number defining the state the Event is currently in: + + - 0 (``INITIALIZED``) - Event is initialized and Freshmaker is searching for artifacts to build. + - 1 (``BUILDING``) - Some artifacts to build have been found and Freshmaker is building them. + - 2 (``COMPLETE``) - All artifacts have been build. Note that this does not mean all the builds were successfull - just one successfull artifact build is enough to mark the Event as ``COMPLETE``. + - 3 (``FAILED``) - There was some major error while handling the Event or all the artifact builds failed to build. + - 4 (``SKIPPED``) - No artifacts have been found to build or the Event is not allowed by the Freshmaker's configuration. + - 5 (``CANCELED``) - Event has been manually cancelled using the REST API. + +.. _event_state_name: + +*state_name* - ``(string)`` + Name of the state the Event is currently in. See ``state`` for more info. + +.. _event_state_reason: + +*state_reason* - ``(string)`` + Sentence describing the current state. + +.. _event_time_created: + +*time_created* - ``(datetime)`` + The date and time on which the Event has been initialized. + +.. _event_time_done: + +*time_done* - ``(datetime)`` + The date and time on which the Event has been moved to ``FAILED``, ``COMPLETE``, or ``CANCELED`` state. + + +.. _build_json_api_1: + +Artifact Build JSON representation +================================== + +The Freshmaker Artifact Build is always represented in the API request as JSON, for example: + +.. sourcecode:: none + + { + "build_args": { + ... + }, + "build_id": 22429387, + "dep_on": "fedora-30-container", + "dep_on_id": 21657, + "event_id": 16730, + "id": 21837, + "name": "httpd-container", + "odcs_composes": [], + "original_nvr": "httpd-container-2.4-1", + "rebuild_reason": "directly_affected", + "rebuilt_nvr": "httpd-container-2.4-1.1561731291", + "state": 1, + "state_name": "DONE", + "state_reason": "Built successfully.", + "time_completed": "2019-06-29T03:28:56Z", + "time_submitted": "2019-06-28T14:14:51Z", + "type": 1, + "type_name": "IMAGE", + "url": "/api/1/builds/21837" + } + +.. _build_build_args: + +*build_args* - ``(JSON)`` + JSON object containing arguments passed to build system to build this artifact. + + .. WARNING:: + The content of this JSON is not part of the Freshmaker REST API and can change at any time. + +.. _build_build_id: + +*build_id* - ``(number)`` + The ID of Artifact Build. + +.. _build_dep_on: + +*dep_on* - ``(string)`` + The :ref:`name`. of the Artifact build this Artifact build depends on. + +.. _build_dep_on_id: + +*dep_on_id* - ``(number)`` + The ID of the Artifact build this Artifact build depends on. + +.. _build_event_id: + +*event_id* - ``(number)`` + The ID of the Event this Artifact Build is part of. + +.. _build_name: + +*name* - ``(string)`` + The name of this Artifact Build. + +.. _build_odcs_composes: + +*odcs_composes* - ``(list of numbers)`` + List of ODCS composes the Freshmaker directly generated and used while building this Artifact build in the build system. + +.. _build_original_nvr: + +*original_nvr* - ``(string)`` + The original (before the rebuild) NVR of Artifact build. + +.. _build_rebuild_reason: + +*rebuild_reason* - ``(string)`` + The reason why this artifact is included in the Event. Can be one of: + + - ``directly_affected`` - The Artifact build is directly affected by the Event (for example affected by the CVE) and is whitelisted by Freshmaker's configuration. + - ``dependency`` - The Artifact build is included in the Event just because it is dependency of ``directly_affected`` Artifact build. + +.. _build_rebuilt_nvr: + +*rebuilt_nvr* - ``(string)`` + The NVR of Artifact build built by Freshmaker. + +.. _build_state: + +*state* - ``(number)`` + Number defining the state the Artifact build is currently in: + + - 0 (``BUILD``) - Artifact build is currently being built in build system. + - 1 (``DONE``) - Artifact build has been built successfully. + - 2 (``FAILED``) - Artifact build failed to be build. + - 3 (``CANCELED``) - Artifact build has been cancelled manually. + - 4 (``PLANNED``) - Artifact build is planned to be build. + +.. _build_state_name: + +*state_name* - ``(string)`` + Name of the state the Artifact build is currently in. See :ref:`state` for more info. + +.. _build_state_reason: + +*state_reason* - ``(string)`` + Sentence describing the current state. + +.. _build_time_completed: + +*time_completed* - ``(datetime)`` + The date and time on which the Artifact Build has been completed. + +.. _build_time_submitted: + +*time_submitted* - ``(datetime)`` + The date and time on which the Artifact build has been moved to ``PLANNED`` state. + +.. _build_type: + +*type* - ``(number)`` + Type of the Artifact build: + + - 0 (``RPM``) - Artifact build is an RPM package. + - 1 (``IMAGE``) - Artifact build is a container image. + - 2 (``MODULE``) - Artifact build is a module. + - 3 (``IMAGE_REPOSITORY``) - Artifact build is a container image repository. + +.. _build_type_name: + +*type__name* - ``(string)`` + Name of the type of Artifact build. See :ref:`type` for more info. + + +.. _pagination_api_1: + +REST API pagination +=================== + +When multiple objects are returned by the Freshmaker REST API, they are wrapped in the following JSON which allows pagination: + +.. sourcecode:: none + + { + "items": [ + {JSON_OBJECT}, + ... + ], + "meta": { + "first": "http://freshmaker.localhost/api/1/events/?per_page=10&page=1", + "last": "http://freshmaker.localhost/api/1/events/?per_page=10&page=14890", + "next": "http://freshmaker.localhost/api/1/events/?per_page=10&page=2", + "page": 1, + "pages": 14890, + "per_page": 10, + "prev": null, + "total": 148898 + } + } + +The ``items`` list contains the objects JSONs. The ``meta`` dict contains metadata about pagination. It is possible to use ``per_page`` argument to set the number of objects showed per single page and ``page`` to choose the page to show. + + +HTTP REST API +============= + +.. automodule:: freshmaker + +.. autoflask:: freshmaker:app + :undoc-static: + :endpoints: event_types_list, event_type, build_types_list, build_type, build_states_list, build_state, events_list, event, builds_list, build, manual_trigger, about, verify_image, verify_image_repository + :modules: freshmaker.views + :order: path diff --git a/docs/api_v2.rst b/docs/api_v2.rst new file mode 100644 index 0000000..25c1639 --- /dev/null +++ b/docs/api_v2.rst @@ -0,0 +1,83 @@ +======================== +API version 2 +======================== + +The Freshmaker API version 2 is mostly same as the API version 1. This document therefore describes only the differences between these two API versions. + +.. _event_json_api_2: + +Event JSON representation +==================================== + +The Freshmaker Event is always represented in the API request as JSON, for example: + +.. sourcecode:: none + + { + "builds_summary": { + "total": 0 + }, + "depending_events": [], + "depends_on_events": [], + "dry_run": false, + "event_type_id": 10, + "id": 18730, + "message_id": "message_123", + "requested_rebuilds": [], + "requester": null, + "requester_metadata": {}, + "search_key": "44637", + "state": 3, + "state_name": "COMPLETE", + "state_reason": "4 images rebuilt.", + "time_created": "2019-08-01T07:00:46Z", + "time_done": "2019-08-01T07:03:12Z", + "url": "/api/1/events/18730" + } + +The meaning of almost all the fields is the same as in :ref:`event_json_api_1` using the API version 1. + +There are following differences between API version 1 and API version 2: + +- The ``builds`` field is replaced with ``builds_summary``. + +.. _event_builds_summary: + +*builds_summary* - ``(JSON)`` + JSON object showing the summary of artifacts builds included in the Event grouped by their ``state_name``. For example: + + .. sourcecode:: none + + "builds_summary": { + "total": 10, + "DONE": 7, + "FAILED": 3 + } + + +.. _build_json_api_2: + +Artifact Build JSON representation +================================== + +The JSON representation of Artifact Build is exactly the same as in :ref:`build_json_api_1` using the API version 1. + + +.. _pagination_api_2: + +REST API pagination +=================== + +The pagination works exactly the same way as in :ref:`pagination_api_1` using the API version 1. + + +HTTP REST API +============= + +.. automodule:: freshmaker + +.. autoflask:: freshmaker:app + :undoc-static: + :endpoints: event_types_list_v2, event_type_v2, build_types_list_v2, build_type_v2, build_states_list_v2, build_state_v2, events_list_v2, event_v2, builds_list_v2, build_v2, manual_trigger_v2, about_v2, verify_image_v2, verify_image_repository_v2 + :modules: freshmaker.views + :order: path diff --git a/docs/index.rst b/docs/index.rst index 416b86c..714fc65 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,7 +15,9 @@ compose (mainly the RPM repository) with packages from Koji using the REST API. :caption: Contents: about - api + api_v1 + api_v2 + messaging_api images_rebuild Indices and tables diff --git a/docs/messaging_api.rst b/docs/messaging_api.rst new file mode 100644 index 0000000..ae47fb9 --- /dev/null +++ b/docs/messaging_api.rst @@ -0,0 +1,20 @@ +======================== +Messaging API +======================== + +Freshmaker also sends AMQP or Fedmsg messages when events or builds change its state. + +``event.state.changed`` +======================= + +This message is sent on every :ref:`Freshmaker Event`'s :ref:`state` change. The message contains :ref:`Event JSON representation as defined in API version 1`. + +``event.state.changed.min`` +=========================== + +This message is sent on every :ref:`Freshmaker Event`'s :ref:`state` change. The message contains :ref:`Event JSON representation as defined in API version 2`. + +``build.state.changed`` +======================= + +This message is sent on every :ref:`Artifact Build`'s :ref:`state` change. The message contains :ref:`Artifact Build`. diff --git a/freshmaker/views.py b/freshmaker/views.py index 1e67220..ab15db2 100644 --- a/freshmaker/views.py +++ b/freshmaker/views.py @@ -224,36 +224,31 @@ class EventAPI(MethodView): @freshmaker_event_api_latency.time() def get(self, id): - """ Returns the list of Freshmaker events. + """ Returns Freshmaker Events. - Whend ``id`` is set, only the Freshmaker Event defined by that ID is + If ``id`` is set, only the Freshmaker Event defined by that ID is returned. - **Sample response**: + :query string message_id: Return only events with this :ref:`message_id`. + :query string search_key: Return only events with this :ref:`search_key`. + :query number event_type_id: Return only events with this :ref:`event_type_id`. + :query number/string state: Return only events int this :ref:`state`. + :query bool show_full_json: When ``True``, the returned Freshmaker Event JSON objects + contains all the fields described in the + :ref:`Freshmaker Event representation for API version 1`. - .. sourcecode:: none + When ``False``, the returned Freshmaker Event JSON objects are in the + :ref:`Freshmaker Event representation for API version 2` format. - { - "builds": [], - "depending_events": [], - "depends_on_events": [], - "dry_run": null, - "event_type_id": 8, - "id": 1000, - "message_id": "ID:message-1", - "requested_rebuilds": [], - "requester": null, - "requester_metadata": {}, - "search_key": "32460", - "state": 4, - "state_name": "SKIPPED", - "state_reason": "Event skipped", - "time_created": "2018-03-06T15:27:49Z", - "time_done": null, - "url": "/api/1/events/1000" - } + Default value for API version 1 is ``True``, for API version 2 is ``False``. + + :query string order_by: Order the events by the given field. If ``-`` prefix is used, + the order will be descending. The default value is ``-id``. Available fields are: - :statuscode 200: Current events are returned. + - :ref:`id` + - :ref:`message_id` + + :statuscode 200: Requested events are returned. :statuscode 404: Freshmaker event not found. """ # Boolean that is set to false if builds should not @@ -297,7 +292,7 @@ class EventAPI(MethodView): Manage Freshmaker event defined by ID. The request must be :mimetype:`application/json`. - Returns the cancelled Freshmaker event. + Returns the cancelled Freshmaker event as JSON. **Sample request**: @@ -311,30 +306,6 @@ class EventAPI(MethodView): "action": "cancel" } - **Sample response**: - - .. sourcecode:: none - - { - "builds": [], - "depending_events": [], - "depends_on_events": [], - "dry_run": null, - "event_type_id": 8, - "id": 1000, - "message_id": "ID:message-1", - "requested_rebuilds": [], - "requester": null, - "requester_metadata": {}, - "search_key": "32460", - "state": 4, - "state_name": "SKIPPED", - "state_reason": "Event skipped", - "time_created": "2018-03-06T15:27:49Z", - "time_done": null, - "url": "/api/1/events/1000" - } - :jsonparam string action: Action to do with an Event. Currently only "cancel" is supported. :statuscode 200: Cancelled event is returned. @@ -404,7 +375,7 @@ class BuildAPI(MethodView): Trigger manual Freshmaker rebuild. The request must be :mimetype:`application/json`. - Returns the newly created Freshmaker event. + Returns the newly created Freshmaker event as JSON. **Sample request**: @@ -418,29 +389,6 @@ class BuildAPI(MethodView): "errata_id": 12345 } - **Sample response**: - - .. sourcecode:: none - - { - "builds": [], - "depending_events": [], - "depends_on_events": [], - "dry_run": null, - "event_type_id": 8, - "id": 1000, - "message_id": "ID:message-1", - "requested_rebuilds": [], - "requester": null, - "requester_metadata": {}, - "search_key": "32460", - "state": 4, - "state_name": "SKIPPED", - "state_reason": "Event skipped", - "time_created": "2018-03-06T15:27:49Z", - "time_done": null, - "url": "/api/1/events/1000" - } :jsonparam string errata_id: The ID of Errata advisory to rebuild artifacts for.