From 2caec6cb7a31a20cc79e16db994c0f4e291737e9 Mon Sep 17 00:00:00 2001 From: Yuxiang Zhu Date: Aug 27 2018 05:10:14 +0000 Subject: CI/CD - Add image promotion pipelines This PR comes with an pipeine template from which users can instantiate image promotion pipelines to promote image between environments. Env files for instantiate pipeline jobs for promoting :dev to :stage and :stage to :prod are also included. A specially designed test suite should be run to ensure the image is mature enough before promotion. Since we currently don't have such a test suite, this pipeline still need to be manually triggered. What this OpenShift Pipeline does is basically: - Pulling the image to be promoted - Pushing the image to destinations with promoted tags. - Optionally tagging the promoted image into an image stream after pushes. NOTE: 1. This pipeline *DOES NOT* cover the step of running tests before actually promoting the image. 2. It's designed to be a callback pipeline triggered by a microservice that handles Greenwave messages. 3. It can be triggered manually (as describe below) to force promoting an image without any tests. Please refer to `openshift/README.md` for instructions. --- diff --git a/openshift/README.md b/openshift/README.md index d11d302..e61e439 100644 --- a/openshift/README.md +++ b/openshift/README.md @@ -191,6 +191,78 @@ oc start-build waiverdb-integration-test \ #### NOTE The stage of reporting test results to ResultsDB has not been implemented. +### Image Promotion Pipeline +Image Promotion Pipeline is a kind of pipeline that promoting an existing image to an upgraded tag. +The pipeline pulls the image to be promoted, then pushes it to destinations with promoted tags. +Optionally, it tags the promoted image into an image stream after pushes. + +NOTE: +1. This pipeline *DOES NOT* cover the step of running tests before actually promoting the image. +2. It's designed to be a callback pipeline triggered by a microservice that handles Greenwave messages. +3. It can be triggered manually (as describe below) to force promoting an image without any tests. + +#### Installation +An OpenShift Template is provided to produce pipelines for promotions between different environments. +You need to instantiate the Template with corresponding parameters for each promotion, like `dev to stage` and `stage to prod`. + +Examples: +- Installing a pipeline for promoting dev image to stage: +```bash +oc process --local -f pipelines/templates/waiverdb-image-promotion-template.yaml \ + -p NAME=waiverdb-promoting-to-stage \ + -p IMAGE="quay.io/factory2/waiverdb:latest" \ + -p PROMOTING_DESTINATIONS="quay.io/factory2/waiverdb:stage,docker-registry.engineering.redhat.com/factory2/waiverdb:stage" \ + | oc apply -f - +``` + +- Installing a pipeline for promoting stage image to prod: +```bash +oc process --local -f pipelines/templates/waiverdb-image-promotion-template.yaml \ + -p NAME=waiverdb-promoting-to-prod \ + -p IMAGE="quay.io/factory2/waiverdb:stage" \ + -p PROMOTING_DESTINATIONS="quay.io/factory2/waiverdb:prod,docker-registry.engineering.redhat.com/factory2/waiverdb:prod" \ + | oc apply -f - +``` + +- Installing a pipeline for promoting dev image to stage with the additional behavior +that automatically tags the image into image stream `waiverdb-stage/waiverdb:stage`: +```bash +oc process --local -f pipelines/templates/waiverdb-image-promotion-template.yaml \ + -p NAME=waiverdb-promoting-to-stage \ + -p IMAGE="quay.io/factory2/waiverdb:latest" \ + -p PROMOTING_DESTINATIONS="quay.io/factory2/waiverdb:stage,docker-registry.engineering.redhat.com/factory2/waiverdb:stage" \ + -p TAG_INTO_IMAGESTREAM=true \ + -p DEST_IMAGESTREAM_NAME=waiverdb \ + -p DEST_IMAGESTREAM_TAG=stage \ + -p DEST_IMAGESTREAM_NAMESPACE=waiverdb-stage \ + | oc apply -f - +``` + +- Installing a pipeline for promoting stage image to prod with the additional behavior +that automatically tags the image into image stream `waiverdb-prod/waiverdb:prod`: +```bash +oc process --local -f ./waiverdb-image-promotion-pipeline-template.yml \ + -p NAME=waiverdb-promoting-to-prod \ + -p IMAGE="quay.io/factory2/waiverdb:stage" \ + -p PROMOTING_DESTINATIONS="quay.io/factory2/waiverdb:prod,docker-registry.engineering.redhat.com/factory2/waiverdb:prod" \ + -p TAG_INTO_IMAGESTREAM=true \ + -p DEST_IMAGESTREAM_NAME=waiverdb \ + -p DEST_IMAGESTREAM_TAG=prod \ + -p DEST_IMAGESTREAM_NAMESPACE=waiverdb-prod \ + | oc apply -f - +``` + +#### Manually Trigger A Promotion +To trigger a promotion, start the corresponding BuildConfig with `oc start-build $PIPELINE_NAME`: + +```bash + # Promoting to stage + oc start-build waiverdb-promoting-to-stage + # Promoting to prod + oc start-build waiverdb-promoting-to-prod +``` +You can go to the OpenShift Web console for more details of the pipeline build. + [OpenShift Pipeline]: https://docs.okd.io/3.9/dev_guide/openshift_pipeline.html [Jenkins Pipeline Build Strategy]: https://docs.openshift.com/container-platform/3.9/dev_guide/dev_tutorials/openshift_pipeline.html [Jenkinsfiles]: https://jenkins.io/doc/book/pipeline/jenkinsfile/ diff --git a/openshift/pipelines/jobs/waiverdb-promoting-to-prod.env b/openshift/pipelines/jobs/waiverdb-promoting-to-prod.env new file mode 100644 index 0000000..7bab6c5 --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-promoting-to-prod.env @@ -0,0 +1,4 @@ +NAME=waiverdb-promoting-to-prod +IMAGE=quay.io/factory2/waiverdb:stage +PROMOTING_DESTINATIONS=quay.io/factory2/waiverdb:prod,docker-registry.engineering.redhat.com/factory2/waiverdb:prod +DEST_IMAGESTREAM_TAG=prod diff --git a/openshift/pipelines/jobs/waiverdb-promoting-to-prod.tmpl b/openshift/pipelines/jobs/waiverdb-promoting-to-prod.tmpl new file mode 100644 index 0000000..7cfc7da --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-promoting-to-prod.tmpl @@ -0,0 +1 @@ +waiverdb-image-promotion-template.yaml diff --git a/openshift/pipelines/jobs/waiverdb-promoting-to-stage.env b/openshift/pipelines/jobs/waiverdb-promoting-to-stage.env new file mode 100644 index 0000000..53bb03d --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-promoting-to-stage.env @@ -0,0 +1,4 @@ +NAME=waiverdb-promoting-to-stage +IMAGE=quay.io/factory2/waiverdb:latest +PROMOTING_DESTINATIONS=quay.io/factory2/waiverdb:stage,docker-registry.engineering.redhat.com/factory2/waiverdb:stage +DEST_IMAGESTREAM_TAG=stage diff --git a/openshift/pipelines/jobs/waiverdb-promoting-to-stage.tmpl b/openshift/pipelines/jobs/waiverdb-promoting-to-stage.tmpl new file mode 100644 index 0000000..7cfc7da --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-promoting-to-stage.tmpl @@ -0,0 +1 @@ +waiverdb-image-promotion-template.yaml diff --git a/openshift/pipelines/templates/waiverdb-image-promotion-template.yaml b/openshift/pipelines/templates/waiverdb-image-promotion-template.yaml new file mode 100644 index 0000000..891c2ff --- /dev/null +++ b/openshift/pipelines/templates/waiverdb-image-promotion-template.yaml @@ -0,0 +1,123 @@ +# Template to produce a pipeline for promoting images between environments +# +# The pipeline pulls the image to be promoted, then pushes it to destinations with promoted tags. +# Optionally, it tags the promoted image into an image stream after pushes. +--- +apiVersion: v1 +kind: Template +metadata: + name: waiverdb-image-promotion +labels: + template: waiverdb-image-promotion +parameters: +- name: NAME + displayName: Short unique identifier for the templated instances + description: This field is used to deploy multiple pipelines to one OpenShift project from this template. + required: true + value: waiverdb-promoting-to-stage-pipeline +- name: IMAGE + displayName: The container image to be promoted + description: This field must be in repo:tag or repo@sha256 format + value: quay.io/factory2/waiverdb:latest +- name: PROMOTING_DESTINATIONS + displayName: Comma seperated list of container repository:tag to which the image will be promoted + description: OpenShift registries must be prefixed with 'atomic:' + required: false + value: "atomic:docker-registry.engineering.redhat.com/factory2/waiverdb:stage,quay.io/factory2/waiverdb:stage" +- name: CONTAINER_REGISTRY_CREDENTIALS + displayName: Secret name of container registries used for pulling and pushing images + value: factory2-pipeline-registry-credentials + required: false +- name: TAG_INTO_IMAGESTREAM + displayName: Whether to tag the image into an ImageStream + value: "false" + required: true +- name: DEST_IMAGESTREAM_NAME + displayName: Name of the ImageStream to be tagged + required: false + value: waiverdb +- name: DEST_IMAGESTREAM_NAMESPACE + displayName: Namespace of the ImageStream to be tagged + description: Leaving blank means using the same namespace as the pipeline build + required: false + value: waiverdb-stage +- name: DEST_IMAGESTREAM_TAG + displayName: Tag name of the ImageStream to be tagged + value: "stage" + required: true +- name: WAIVERDB_GIT_REPO + displayName: WaiverDB Git repo URL + description: Default WaiverDB Git repo URL in which to run functional tests against + required: true + value: "https://pagure.io/waiverdb.git" +- name: WAIVERDB_GIT_REF + displayName: WaiverDB Git repo ref + description: Default WaiverDB Git repo ref in which to run functional tests against + required: true + value: master +- name: JENKINS_AGENT_IMAGE + displayName: Container image for Jenkins slave pods + required: true + value: docker-registry.engineering.redhat.com/factory2/waiverdb-jenkins-slave:latest +- name: JENKINS_AGENT_CLOUD_NAME + displayName: Name of OpenShift cloud in Jenkins master configuration + required: true + value: openshift +objects: +- kind: ServiceAccount + apiVersion: v1 + metadata: + name: "${NAME}-jenkins-slave" + labels: + app: "${NAME}" +- kind: RoleBinding + apiVersion: v1 + metadata: + name: "${NAME}-jenkins-slave_edit" + labels: + app: "${NAME}" + subjects: + - kind: ServiceAccount + name: "${NAME}-jenkins-slave" + roleRef: + name: edit +- kind: "BuildConfig" + apiVersion: "v1" + metadata: + name: "${NAME}" + labels: + app: "${NAME}" + spec: + runPolicy: "Serial" # FIXME: Parallel is supported, but we have limited quota in UpShift. + completionDeadlineSeconds: 1800 + source: + git: + uri: "${WAIVERDB_GIT_REPO}" + ref: "${WAIVERDB_GIT_REF}" + strategy: + type: JenkinsPipeline + source: + type: None + jenkinsPipelineStrategy: + env: + - name: "IMAGE" + value: "${IMAGE}" + - name: "PROMOTING_DESTINATIONS" + value: "${PROMOTING_DESTINATIONS}" + - name: "CONTAINER_REGISTRY_CREDENTIALS" + value: "${CONTAINER_REGISTRY_CREDENTIALS}" + - name: "TAG_INTO_IMAGESTREAM" + value: "${TAG_INTO_IMAGESTREAM}" + - name: "DEST_IMAGESTREAM_NAME" + value: "${DEST_IMAGESTREAM_NAME}" + - name: "DEST_IMAGESTREAM_NAMESPACE" + value: "${DEST_IMAGESTREAM_NAMESPACE}" + - name: "DEST_IMAGESTREAM_TAG" + value: "${DEST_IMAGESTREAM_TAG}" + - name: JENKINS_AGENT_IMAGE + value: "${JENKINS_AGENT_IMAGE}" + - name: JENKINS_AGENT_CLOUD_NAME + value: "${JENKINS_AGENT_CLOUD_NAME}" + - name: JENKINS_AGENT_SERVICE_ACCOUNT + value: "${NAME}-jenkins-slave" + jenkinsfilePath: openshift/pipelines/templates/waiverdb-image-promotion.Jenkinsfile diff --git a/openshift/pipelines/templates/waiverdb-image-promotion.Jenkinsfile b/openshift/pipelines/templates/waiverdb-image-promotion.Jenkinsfile new file mode 100644 index 0000000..d6f27a0 --- /dev/null +++ b/openshift/pipelines/templates/waiverdb-image-promotion.Jenkinsfile @@ -0,0 +1,127 @@ +pipeline { + agent { + kubernetes { + cloud "${params.JENKINS_AGENT_CLOUD_NAME}" + label "jenkins-slave-${UUID.randomUUID().toString()}" + serviceAccount "${params.JENKINS_AGENT_SERVICE_ACCOUNT}" + defaultContainer 'jnlp' + yaml """ + apiVersion: v1 + kind: Pod + metadata: + labels: + app: "jenkins-${env.JOB_BASE_NAME}" + factory2-pipeline-kind: "waiverdb-image-promotion-pipeline" + factory2-pipeline-build-number: "${env.BUILD_NUMBER}" + spec: + containers: + - name: jnlp + image: "${params.JENKINS_AGENT_IMAGE}" + imagePullPolicy: Always + tty: true + env: + - name: REGISTRY_CREDENTIALS + valueFrom: + secretKeyRef: + name: "${params.CONTAINER_REGISTRY_CREDENTIALS}" + key: '.dockerconfigjson' + resources: + requests: + memory: 512Mi + cpu: 200m + limits: + memory: 768Mi + cpu: 500m + """ + } + } + options { + timestamps() + timeout(time: 30, unit: 'MINUTES') + } + environment { + PIPELINE_NAMESPACE = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/namespace').trim() + SERVICE_ACCOUNT_TOKEN = readFile(file: '/var/run/secrets/kubernetes.io/serviceaccount/token').trim() + } + stages { + stage ('Prepare') { + steps { + script { + // Setting up registry credentials + dir ("${env.HOME}/.docker") { + // for the OpenShift internal registry + def dockerConfig = readJSON text: '{ "auths": {} }' + dockerConfig.auths['docker-registry.default.svc:5000'] = [ + 'email': '', + 'auth': sh(returnStdout: true, script: 'set +x; echo -n "serviceaccount:$SERVICE_ACCOUNT_TOKEN" | base64 -').trim() + ] + // merging user specified credentials + if (env.REGISTRY_CREDENTIALS) { + toBeMerged = readJSON text: env.REGISTRY_CREDENTIALS + dockerConfig.auths.putAll(toBeMerged.auths) + } + // writing to ~/.docker/config.json + writeJSON file: 'config.json', json: dockerConfig + } + } + } + } + stage('Pull Container') { + steps { + echo "Pulling container image ${params.IMAGE}..." + sh '''set -e +x # hide the token from Jenkins console + rm -rf _build/container + mkdir -p _build + skopeo copy \ + --src-cert-dir=/var/run/secrets/kubernetes.io/serviceaccount \ + docker://"$IMAGE" dir:_build/container + ''' + } + } + stage('Promote') { + steps { + script { + def destinations = params.PROMOTING_DESTINATIONS ? + params.PROMOTING_DESTINATIONS.split(',') : [] + openshift.withCluster() { + def pushTasks = destinations.collectEntries { + ["Pushing ${it}" : { + def dest = it + // Only docker and atomic registries are allowed + if (!it.startsWith('atomic:') && !it.startsWith('docker://')) { + dest = 'docker://' + it + } + echo "Pushing container to ${dest}..." + withEnv(["DEST_IMAGE_REF=${dest}"]) { + /* Pushes to the internal registry can sometimes randomly fail + * with "unknown blob" due to a known issue with the registry + * storage configuration. So we retry up to 5 times. */ + retry(5) { + sh 'skopeo copy dir:_build/container "$DEST_IMAGE_REF"' + } + } + }] + } + parallel pushTasks + } + } + } + } + stage('Tag Image Stream') { + when { + expression { + return params.DEST_IMAGESTREAM_NAME && params.TAG_INTO_IMAGESTREAM == "true" + } + } + steps { + script { + def destRef = "${params.DEST_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_NAMESPACE }/${params.DEST_IMAGESTREAM_NAME}:${params.DEST_IMAGESTREAM_TAG}" + openshift.withCluster() { + echo "Tagging ${params.IMAGE} into ${destRef}..." + openshift.tag('--source=docker', params.IMAGE, destRef) + } + } + } + } + } +}