From b7494b5fbe96ffaeb4c0375f05def53e05824fb2 Mon Sep 17 00:00:00 2001 From: Yuxiang Zhu Date: Mar 07 2019 18:06:22 +0000 Subject: CI/CD: minor improvements 1. Append UUID to TEST_ID for integration tests to avoid name conflicts 2. Set time limit for Pagure polling job to 30 min 3. Ignore errors of installing dependencies from requirements.txt 4. Ignore failures of cleaning old test environment deployments 5. Do not push to the internal registry (we got enough 'Unknown blob' errors) 6. Fix waiverdb-test-template doesn't support pulling by digests 7. Change test case name format --- diff --git a/Jenkinsfile b/Jenkinsfile index f9a65fb..380c7c7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -181,7 +181,7 @@ node('fedora-28') { def template = readYaml file: 'openshift/waiverdb-test-template.yaml' def models = openshift.process(template, '-p', "TEST_ID=${env.BUILD_TAG}", - '-p', "WAIVERDB_APP_VERSION=internal-${appversion}") + '-p', "WAIVERDB_APP_IMAGE=quay.io/factory2/waiverdb:internal-${appversion}") def environment_label = "test-${env.BUILD_TAG}" try { def objects = openshift.create(models) diff --git a/openshift/pipelines/templates/waiverdb-build-template.yaml b/openshift/pipelines/templates/waiverdb-build-template.yaml index a2fd8cb..998ce68 100644 --- a/openshift/pipelines/templates/waiverdb-build-template.yaml +++ b/openshift/pipelines/templates/waiverdb-build-template.yaml @@ -67,7 +67,7 @@ parameters: displayName: Comma seperated list of container repositories (without tag) to which the built WaiverDB dev image will be pushed description: OpenShift registries must be prefixed with 'atomic:' required: false - value: "quay.io/factory2/waiverdb,atomic:docker-registry.engineering.redhat.com/factory2/waiverdb" + value: "quay.io/factory2/waiverdb" - name: CONTAINER_REGISTRY_CREDENTIALS displayName: Secret name of container registries used for pulling and pushing images value: factory2-pipeline-registry-credentials diff --git a/openshift/pipelines/templates/waiverdb-build.Jenkinsfile b/openshift/pipelines/templates/waiverdb-build.Jenkinsfile index 244d254..7425d86 100644 --- a/openshift/pipelines/templates/waiverdb-build.Jenkinsfile +++ b/openshift/pipelines/templates/waiverdb-build.Jenkinsfile @@ -107,6 +107,10 @@ pipeline { env.WAIVERDB_VERSION = versions[0] env.WAIVERDB_CONTAINER_VERSION = versions[1] env.TEMP_TAG = env.WAIVERDB_CONTAINER_VERSION + '-jenkins-' + currentBuild.id + + if (sh(returnStatus: true, script: 'pip3 install --user -r ./requirements.txt' != 0) { + echo 'WARNING: Failed to install dependencies from requirements.txt.' + } } } } @@ -144,8 +148,6 @@ pipeline { currentBuild.description = """${currentBuild.displayName}""" } } - sh 'cp conf/settings.py.example conf/settings.py' - sh 'pip3 install --user -r ./requirements.txt' } } stage('Run checks') { @@ -165,6 +167,7 @@ pipeline { } stage('Run unit tests') { steps { + sh 'cp conf/settings.py.example conf/settings.py' // wait for the test datebase to come up sh 'wait-for-it -s -t 300 127.0.0.1:5432' // create a database role diff --git a/openshift/pipelines/templates/waiverdb-integration-test-template.yaml b/openshift/pipelines/templates/waiverdb-integration-test-template.yaml index bc7d168..17021ee 100644 --- a/openshift/pipelines/templates/waiverdb-integration-test-template.yaml +++ b/openshift/pipelines/templates/waiverdb-integration-test-template.yaml @@ -88,7 +88,7 @@ objects: - name: "IMAGE" value: "${IMAGE}" - name: IMAGE_IS_SCRATCH - value: "false" + value: "true" - name: "CONTAINER_REGISTRY_CREDENTIALS" value: "${CONTAINER_REGISTRY_CREDENTIALS}" - name: "TEST_ID" diff --git a/openshift/pipelines/templates/waiverdb-integration-test.Jenkinsfile b/openshift/pipelines/templates/waiverdb-integration-test.Jenkinsfile index b602cac..ff33706 100644 --- a/openshift/pipelines/templates/waiverdb-integration-test.Jenkinsfile +++ b/openshift/pipelines/templates/waiverdb-integration-test.Jenkinsfile @@ -71,8 +71,12 @@ pipeline { if (now.getTime() - creationTime.getTime() < 1000 * 60 * 60) continue echo "Deleting ${objName}..." - obj.delete() - echo "Deleted ${objName}" + try { + obj.delete() + echo "Deleted ${objName}" + } catch (e) { + echo "Error deleting ${objName}: ${e.message}" + } } } } @@ -81,22 +85,20 @@ pipeline { stage('Run functional tests') { environment { // Jenkins BUILD_TAG could be too long (> 63 characters) for OpenShift to consume - TEST_ID = "${params.TEST_ID ?: 'jenkins-' + currentBuild.id}" - ENVIRONMENT_LABEL = "test-${env.TEST_ID}" + TEST_ID = "${params.TEST_ID ?: 'jenkins-' + currentBuild.id + '-' + UUID.randomUUID().toString().substring(0,7)}" } steps { echo "Container image ${params.IMAGE} will be tested." script { openshift.withCluster() { - def imageTag = (params.IMAGE =~ /(?::(\w[\w.-]{0,127}))?$/)[0][1] - def imageRepo = imageTag ? params.IMAGE.substring(0, params.IMAGE.length() - imageTag.length() - 1) : params.IMAGE - env.IMAGE_TAG = imageTag ?: 'latest' + // Don't set ENVIRONMENT_LABEL in the environment block! Otherwise you will get 2 different UUIDs. + env.ENVIRONMENT_LABEL = "test-${env.TEST_ID}" def template = readYaml file: 'openshift/waiverdb-test-template.yaml' def webPodReplicas = 1 // The current quota in UpShift is agressively limited + echo "Creating testing environment with TEST_ID=${env.TEST_ID}..." def models = openshift.process(template, '-p', "TEST_ID=${env.TEST_ID}", - '-p', "WAIVERDB_APP_IMAGE_REPO=${imageRepo}", - '-p', "WAIVERDB_APP_VERSION=${imageTag ?: 'latest'}", + '-p', "WAIVERDB_APP_IMAGE=${params.IMAGE}", '-p', "WAIVERDB_REPLICAS=${webPodReplicas}", ) def objects = openshift.apply(models) @@ -193,7 +195,7 @@ pipeline { "type": "container-image", "repository": "factory2/waiverdb", "digest": "${env.IMAGE_DIGEST}", - "nvr": "waiverdb:${env.IMAGE_TAG}", + "nvr": "${params.IMAGE}", "issuer": "c3i-jenkins", "scratch": ${params.IMAGE_IS_SCRATCH}, "id": "waiverdb@${env.IMAGE_DIGEST}" @@ -204,8 +206,8 @@ pipeline { "provider": "openshift", "architecture": "x86_64" }], - "type": "${params.ENVIRONMENT}", - "category": "integration", + "type": "integration", + "category": "${params.ENVIRONMENT}", "status": "${currentBuild.result == null || currentBuild.result == 'SUCCESS' ? 'passed':'failed'}", "xunit": "${env.BUILD_URL}/artifacts/junit-functional-tests.xml", "generated_at": "${new Date().format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone('UTC'))}", diff --git a/openshift/pipelines/templates/waiverdb-polling-pagure.yaml b/openshift/pipelines/templates/waiverdb-polling-pagure.yaml index c1694bc..137ce28 100644 --- a/openshift/pipelines/templates/waiverdb-polling-pagure.yaml +++ b/openshift/pipelines/templates/waiverdb-polling-pagure.yaml @@ -84,6 +84,7 @@ objects: app: "${NAME}" spec: runPolicy: "Serial" + completionDeadlineSeconds: 1800 strategy: type: JenkinsPipeline jenkinsPipelineStrategy: diff --git a/openshift/waiverdb-test-template.yaml b/openshift/waiverdb-test-template.yaml index f200a80..1627b3e 100644 --- a/openshift/waiverdb-test-template.yaml +++ b/openshift/waiverdb-test-template.yaml @@ -4,7 +4,7 @@ # for Postgres data. # # To create an environment from the template, process and apply it: -# oc process -f openshift/waiverdb-test-template.yaml -p TEST_ID=123 -p WAIVERDB_APP_VERSION=0.1.2.dev24-git.94c0119 | oc apply -f - +# oc process -f openshift/waiverdb-test-template.yaml -p TEST_ID=123 -p WAIVERDB_APP_IMAGE=quay.io/factory2/waiverdb:0.1.2.dev24-git.94c0119 | oc apply -f - # To clean up the environment, use a selector on the environment label: # oc delete dc,deploy,pod,configmap,secret,svc,route -l environment=test-123 @@ -20,14 +20,10 @@ parameters: displayName: Test id description: Short unique identifier for this test run (e.g. Jenkins job number) required: true -- name: WAIVERDB_APP_VERSION - displayName: WaiverDB application version - description: Python version of the WaiverDB application being tested +- name: WAIVERDB_APP_IMAGE + displayName: The image of WaiverDB application being tested required: true -- name: WAIVERDB_APP_IMAGE_REPO - displayName: The registry/repo of WaiverDB application being tested - required: true - value: docker-registry.engineering.redhat.com/factory2/waiverdb + value: quay.io/factory2/waiverdb:latest - name: FLASK_SECRET_KEY displayName: Flask secret key generate: expression @@ -223,7 +219,7 @@ objects: spec: containers: - name: web - image: "${WAIVERDB_APP_IMAGE_REPO}:${WAIVERDB_APP_VERSION}" + image: "${WAIVERDB_APP_IMAGE}" ports: - containerPort: 8080 volumeMounts: