From c468465cf656c03ea61eb68ca71b2ead90ee5650 Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Oct 29 2021 13:35:00 +0000 Subject: [PATCH 1/2] using kube-sig operator image --- diff --git a/operator/Dockerfile b/operator/Dockerfile index 7d3f4a1..496b2e7 100644 --- a/operator/Dockerfile +++ b/operator/Dockerfile @@ -1,11 +1,9 @@ -FROM quay.io/operator-framework/ansible-operator:v1.7.2 +FROM quay.io/fedora-kube-sig/ansible-operator:latest USER root -RUN dnf install -y git gcc libpq libpq-devel python38-devel krb5-devel wget python38-psycopg2 -#TODO: find a better way to install both koji and cloud-init and its schema.sql file -RUN pip3.8 install koji==1.25.1 git+https://github.com/canonical/cloud-init.git@20.4.1 -COPY hack/schema.sql /usr/share/doc/koji/docs/schema.sql -RUN chmod 644 /usr/share/doc/koji/docs/schema.sql +RUN dnf install -y --setopt='fastestmirror=1' \ + git gcc libpq libpq-devel krb5-devel wget python3-psycopg2 cloud-init +RUN dnf install --setopt='fastestmirror=1' --setopt='tsflags=' -y koji USER ${USER_ID} COPY requirements.yml ${HOME}/requirements.yml From cacdf5171fd3f7521220565471bf8fd0bca1fd51 Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Nov 04 2021 03:16:07 +0000 Subject: [PATCH 2/2] kubevirt readiness check --- diff --git a/operator/config/rbac/kubevirt_editor_role.yaml b/operator/config/rbac/kubevirt_editor_role.yaml new file mode 100644 index 0000000..4dd15a5 --- /dev/null +++ b/operator/config/rbac/kubevirt_editor_role.yaml @@ -0,0 +1,27 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: vmi-editor-role +rules: +- apiGroups: + - kubevirt.io + resources: + - virtualmachineinstances + - virtualmachines + - virtualmachineinstancepresets + - virtualmachineinstancereplicasets + verbs: + - get + - delete + - create + - update + - patch + - list + - watch + - deletecollection +- apiGroups: + - kubevirt.io + resources: + - virtualmachineinstances/status + verbs: + - get diff --git a/operator/config/rbac/kubevirt_viewer_role.yaml b/operator/config/rbac/kubevirt_viewer_role.yaml new file mode 100644 index 0000000..225ad64 --- /dev/null +++ b/operator/config/rbac/kubevirt_viewer_role.yaml @@ -0,0 +1,29 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: vmi-viewer-role +rules: +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/console + - virtualmachineinstances/vnc + verbs: + - get +- apiGroups: + - kubevirt.io + resources: + - virtualmachineinstances + - virtualmachines + - virtualmachineinstancepresets + - virtualmachineinstancereplicasets + verbs: + - get + - list + - watch +- apiGroups: + - kubevirt.io + resources: + - virtualmachineinstances/status + verbs: + - get diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index ddae2eb..e3ccaa5 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -3,6 +3,8 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: manager-role + labels: + kubevirt.io: "" rules: ## ## Base operator rules @@ -120,20 +122,31 @@ rules: - update - watch ## - ## Rules for kubevirt.io/v1alpha3, Kind: VirtualMachineInstance + ## Rules for the apiGroup kubevirt.io ## - apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/console + - virtualmachineinstances/vnc + verbs: + - get + - apiGroups: - kubevirt.io resources: + - virtualmachineinstances - virtualmachines + - virtualmachineinstancepresets + - virtualmachineinstancereplicasets - virtualmachines/status - virtualmachines/finalizers verbs: - - create - - delete - get - - list - - patch + - delete + - create - update + - patch + - list - watch + - deletecollection #+kubebuilder:scaffold:rules diff --git a/operator/molecule/default/tasks/kojibuilder_kubevirt_test.yml b/operator/molecule/default/tasks/kojibuilder_kubevirt_test.yml index 207a135..b403406 100644 --- a/operator/molecule/default/tasks/kojibuilder_kubevirt_test.yml +++ b/operator/molecule/default/tasks/kojibuilder_kubevirt_test.yml @@ -22,15 +22,12 @@ volume: false running: true wait: yes - wait_timeout: 300 + wait_timeout: 1800 wait_condition: type: Running reason: Successful status: "True" -- name: Pause until you can verify updates to an application were successful - pause: - - block: - name: 'TEST: kojibuilder.configmap' k8s_info: @@ -58,69 +55,39 @@ - kojibuilder_client_secrets.resources|length == 1 - kojibuilder_client_secrets.resources[0].metadata.labels['app'] == 'koji-builder' - "'client.pem' in kojibuilder_client_secrets.resources[0].data" -- block: - - name: 'TEST: kojibuilder.secret.ssh-key' - k8s_info: - api_version: v1 - kind: Secret - namespace: "{{ namespace }}" - name: "kojid-sample-virt-default-ssh-key" - register: kojibuilder_ssh_secrets - - assert: - that: - - kojibuilder_ssh_secrets.resources|length == 1 - - kojibuilder_ssh_secrets.resources[0].metadata.labels['app'] == 'koji-builder' - - kojibuilder_ssh_secrets.resources[0].metadata.labels['ctx'] == 'ssh-key' - - kojibuilder_ssh_secrets.resources[0].metadata.labels['kojid'] == 'sample-virt' - - "'id_rsa' in kojibuilder_ssh_secrets.resources[0].data" - - "'id_rsa.pub' in kojibuilder_ssh_secrets.resources[0].data" -- block: - - name: 'TEST: kojibuilder.secret.ssh-pub-keys' - k8s_info: - api_version: v1 - kind: Secret - namespace: "{{ namespace }}" - name: kojid-sample-virt-ssh-pub-keys - register: kojibuilder_ssh_pub_secrets - - assert: - that: - - kojibuilder_ssh_pub_secrets.resources|length == 1 - - kojibuilder_ssh_pub_secrets.resources[0].metadata.labels['app'] == 'koji-builder' - - kojibuilder_ssh_pub_secrets.resources[0].metadata.labels['ctx'] == 'ssh-key-pub' - - kojibuilder_ssh_pub_secrets.resources[0].metadata.labels['kojid'] == 'sample-virt' - - "'id_rsa' not in kojibuilder_ssh_pub_secrets.resources[0].data" - - "'id_rsa.pub' not in kojibuilder_ssh_pub_secrets.resources[0].data" - - "'key1' in kojibuilder_ssh_pub_secrets.resources[0].data" -- block: - - name: 'TEST: kojibuilder.configmap.cloudinit' - k8s_info: - api_version: v1 - kind: Secret - namespace: "{{ namespace }}" - name: kojid-sample-virt-cloudinit-conf - register: kojibuilder_cloudinit_secret - - debug: var=kojibuilder_cloudinit_secret - - assert: - that: - - kojibuilder_cloudinit_secret.resources|length == 1 - - "kojibuilder_cloudinit_secret.resources[0].metadata.labels['app'] == 'kojid-sample-virt-kubevirt'" - - "'userData' in kojibuilder_cloudinit_secret.resources[0].data" - -- block: - - name: 'TEST: kojibuilder.configmap.kojid-default-setup' - k8s_info: - api_version: v1 - kind: ConfigMap - namespace: "{{ namespace }}" - name: kojid-default-setup - register: kojibuilder_setup_cm - - debug: var=kojibuilder_setup_cm - - assert: - that: - - kojibuilder_setup_cm.resources|length == 1 - - "'setup.sh' in kojibuilder_setup_cm.resources[0].data" - - "'kojid.service' in kojibuilder_setup_cm.resources[0].data" - + #- block: + # - name: 'TEST: kojibuilder.secret.ssh-key' + # k8s_info: + # api_version: v1 + # kind: Secret + # namespace: "{{ namespace }}" + # name: "kojid-sample-virt-default-ssh-key" + # register: kojibuilder_ssh_secrets + # - assert: + # that: + # - kojibuilder_ssh_secrets.resources|length == 1 + # - kojibuilder_ssh_secrets.resources[0].metadata.labels['app'] == 'koji-builder' + # - kojibuilder_ssh_secrets.resources[0].metadata.labels['ctx'] == 'ssh-key' + # - kojibuilder_ssh_secrets.resources[0].metadata.labels['kojid'] == 'sample-virt' + # - "'id_rsa' in kojibuilder_ssh_secrets.resources[0].data" + # - "'id_rsa.pub' in kojibuilder_ssh_secrets.resources[0].data" + #- block: + # - name: 'TEST: kojibuilder.secret.ssh-pub-keys' + # k8s_info: + # api_version: v1 + # kind: Secret + # namespace: "{{ namespace }}" + # name: kojid-sample-virt-ssh-pub-keys + # register: kojibuilder_ssh_pub_secrets + # - assert: + # that: + # - kojibuilder_ssh_pub_secrets.resources|length == 1 + # - kojibuilder_ssh_pub_secrets.resources[0].metadata.labels['app'] == 'koji-builder' + # - kojibuilder_ssh_pub_secrets.resources[0].metadata.labels['ctx'] == 'ssh-key-pub' + # - kojibuilder_ssh_pub_secrets.resources[0].metadata.labels['kojid'] == 'sample-virt' + # - "'id_rsa' not in kojibuilder_ssh_pub_secrets.resources[0].data" + # - "'id_rsa.pub' not in kojibuilder_ssh_pub_secrets.resources[0].data" + # - "'key1' in kojibuilder_ssh_pub_secrets.resources[0].data" - block: - name: 'TEST: kojibuilder.deletion' k8s: diff --git a/operator/molecule/minikube/create.yml b/operator/molecule/minikube/create.yml index 405f654..b6b82d3 100644 --- a/operator/molecule/minikube/create.yml +++ b/operator/molecule/minikube/create.yml @@ -15,6 +15,17 @@ - block: - name: Enable kubevirt command: minikube addons enable kubevirt + - name: Wait for kubevirt namespace + k8s_info: + api_version: v1 + kind: Namespace + name: kubevirt + register: out + retries: 180 + delay: 3 + until: + - "out.resources | length == 1" + - k8s_info: api_version: kubevirt.io/v1 kind: KubeVirt diff --git a/operator/roles/koji-builder/files/cloud-init-config.yaml b/operator/roles/koji-builder/files/cloud-init-config.yaml index 80c8d75..ac5c168 100644 --- a/operator/roles/koji-builder/files/cloud-init-config.yaml +++ b/operator/roles/koji-builder/files/cloud-init-config.yaml @@ -23,4 +23,7 @@ bootcmd: # mockcfg - "mkdir -p /mnt/mockcfg" - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK270D | cut -f1 -d' ') /mnt/mockcfg" + # status server + - "mkdir -p /mnt/cloud-init-status-server" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK280D | cut -f1 -d' ') /mnt/cloud-init-status-server" diff --git a/operator/roles/koji-builder/files/cloud-init-script.sh b/operator/roles/koji-builder/files/cloud-init-script.sh index 1c651ca..beb2cea 100644 --- a/operator/roles/koji-builder/files/cloud-init-script.sh +++ b/operator/roles/koji-builder/files/cloud-init-script.sh @@ -1,7 +1,14 @@ #!/bin/bash + +# status server +mkdir -p /opt/cloud-init-status-server +ln -sf /var/lib/cloud/data/result.json /opt/cloud-init-status-server/result.json +cp /mnt/cloud-init-status-server/cloud-init-status-server.service /usr/lib/systemd/system/cloud-init-status-server.service +systemctl enable cloud-init-status-server +systemctl start cloud-init-status-server # packages -dnf install -y --setopt='fastestmirror=True' koji-builder +dnf install -y --setopt='fastestmirror=True' koji-builder python3-gunicorn python3-flask #kojid setup mkdir /mnt/kojid diff --git a/operator/roles/koji-builder/tasks/builders/kubevirt/create.yml b/operator/roles/koji-builder/tasks/builders/kubevirt/create.yml index 6bb0a1f..590603a 100644 --- a/operator/roles/koji-builder/tasks/builders/kubevirt/create.yml +++ b/operator/roles/koji-builder/tasks/builders/kubevirt/create.yml @@ -7,17 +7,6 @@ channels: "{{ koji_builder_channels }}" koji: "{{ koji_builder_hub_name }}-{{ koji_builder_hub_namespace }}" -- name: Check if ssh key is already present - kubernetes.core.k8s_info: - kind: Secret - namespace: "{{ meta.namespace }}" - name: "kojid-{{ meta.name }}-ssh-pub-keys" - register: k8s_out - -- name: Set k8s_secret var - set_fact: - k8s_secret: "{{ k8s_out.resources | length > 0 }}" - - name: create temporary cert directory tempfile: state: directory @@ -25,93 +14,42 @@ suffix: kubevirt register: cert_dir -- name: Create ssh private key - community.crypto.openssh_keypair: - path: "{{ cert_dir.path }}/key.pem" - size: 4096 - register: koji_builer_ssh_key - when: not k8s_secret - -- name: Create default ssh secret - k8s: - definition: - apiVersion: v1 - kind: Secret - metadata: - namespace: "{{ meta.namespace }}" - name: "kojid-{{ meta.name }}-default-ssh-key" - labels: - app: koji-builder - ctx: ssh-key - kojid: "{{ meta.name }}" - data: - id_rsa: "{{ lookup('file', koji_builer_ssh_key.filename) | b64encode }}" - id_rsa.pub: "{{ koji_builer_ssh_key.public_key | b64encode }}" - when: not k8s_secret - -- name: Create public ssh secret - k8s: - definition: - apiVersion: v1 - kind: Secret - metadata: - namespace: "{{ meta.namespace }}" - name: "kojid-{{ meta.name }}-ssh-pub-keys" - labels: - app: koji-builder - ctx: ssh-key-pub - kojid: "{{ meta.name }}" - data: - key1: "{{ koji_builer_ssh_key.public_key | b64encode }}" - when: not k8s_secret - -- name: Expose kubevirt ssh service - kubernetes.core.k8s: - definition: "{{ lookup('template', 'kubevirt/service.yaml.j2') | from_yaml }}" - wait: true - state: present - - #- name: Create cloudinit config - # kubernetes.core.k8s: - # definition: "{{ lookup('template', 'kubevirt/cloud-init.yaml.j2') | from_yaml }}" - # wait: true - # state: present - # vars: - # cinit_kojid_contents: "{{ k8s_kojid_cm.resources[0].data['kojid.conf'] }}" - - name: Create mock pvc kubernetes.core.k8s: definition: "{{ lookup('template', 'kubevirt/pvc.yaml.j2') | from_yaml }}" wait: true state: present - #- name: Ensure default setup script is present - # kubernetes.core.k8s: - # definition: "{{ lookup('template', 'kubevirt/default-setup-configmap.yaml.j2') | from_yaml }}" - # wait: true - # state: present - -- name: Generate cloud-init mult part file data - koji_cloudinit_userdata: - user_config: "{{ role_path }}/files/cloud-init-config.yaml" - user_script: "{{ role_path }}/files/cloud-init-script.sh" - register: koji_builder_cloud_init_user_data +- name: Check if bei + kubernetes.core.k8s_info: + kind: Secret + namespace: "{{ meta.namespace }}" + name: "kojid-{{ meta.name }}-cloud-init" + register: k8s_out -- name: Create cloud-init configuration secret - k8s: - state: present - wait: true - definition: - apiVersion: v1 - kind: Secret - metadata: - annotations: - cloud-init-config: "{{ lookup('file', 'cloud-init-config.yaml') }}" - cloud-init-script: "{{ lookup('file', 'cloud-init-script.sh') }}" - namespace: "{{ meta.namespace }}" - name: "kojid-{{ meta.name }}-cloud-init" - data: - userData: "{{ koji_builder_cloud_init_user_data.data | b64encode }}" +- block: + - name: Generate cloud-init mult part file data + koji_cloudinit_userdata: + user_config: "{{ role_path }}/files/cloud-init-config.yaml" + user_script: "{{ role_path }}/files/cloud-init-script.sh" + register: koji_builder_cloud_init_user_data + + - name: Create cloud-init configuration secret + k8s: + state: present + wait: true + definition: + apiVersion: v1 + kind: Secret + metadata: + annotations: + cloud-init-config: "{{ lookup('file', 'cloud-init-config.yaml') }}" + cloud-init-script: "{{ lookup('file', 'cloud-init-script.sh') }}" + namespace: "{{ meta.namespace }}" + name: "kojid-{{ meta.name }}-cloud-init" + data: + userData: "{{ koji_builder_cloud_init_user_data.data | b64encode }}" + when: k8s_out.resources | length == 0 - name: Create mock config file k8s: @@ -128,17 +66,22 @@ data: site-defaults.cfg: "{{ lookup('file', 'mock-site-defaults.cfg') }}" -- block: - - name: - template: - src: kubevirt/vmi.yaml.j2 - dest: "{{ koji_dir_path}}/vmi.yaml" +- name: Create cloud init status script configmap + k8s: + state: present + wait: true + definition: "{{ lookup('template', 'kubevirt/cloud-init-status-server.configmap.yaml.j2') }}" + wait: true + +- name: Apply k8s vm resource + kubernetes.core.k8s: + definition: "{{ lookup('template', 'kubevirt/vm.yaml.j2') | from_yaml }}" + state: present + wait: true + wait_condition: + type: Ready + status: "True" - - name: Apply k8s vmi resource - k8s: - state: present - src: "{{ koji_dir_path}}/vmi.yaml" - - name: Cleanup file: path: "{{ cert_dir.path }}" diff --git a/operator/roles/koji-builder/templates/kubevirt/cloud-init-status-server.configmap.yaml.j2 b/operator/roles/koji-builder/templates/kubevirt/cloud-init-status-server.configmap.yaml.j2 new file mode 100644 index 0000000..3bb0bff --- /dev/null +++ b/operator/roles/koji-builder/templates/kubevirt/cloud-init-status-server.configmap.yaml.j2 @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kojid-cloudinit-status-server + namespace: "{{ meta.namespace }}" + labels: + app: kojid +data: + cloud-init-status-server.service: |- + [Unit] + Description=Cloud Init Status Server + + After=network.target + + [Service] + ExecStart=python3 \ + -m http.server \ + 5000 \ + -d /opt/cloud-init-status-server \ + --bind 0.0.0.0 + ExecReload=/bin/kill -USR1 $MAINPID + + [Install] + WantedBy=multi-user.target diff --git a/operator/roles/koji-builder/templates/kubevirt/vm.yaml.j2 b/operator/roles/koji-builder/templates/kubevirt/vm.yaml.j2 new file mode 100644 index 0000000..8acee76 --- /dev/null +++ b/operator/roles/koji-builder/templates/kubevirt/vm.yaml.j2 @@ -0,0 +1,83 @@ +apiVersion: kubevirt.io/v1alpha3 +kind: VirtualMachine +metadata: + name: "kojid-{{ meta.name }}-{{ koji_builder_type }}" + labels: + app: "kojid-{{ meta.name }}-{{ koji_builder_type }}" + namespace: {{ meta.namespace }} +spec: + running: {{ koji_builder_running }} + template: + metadata: + labels: + kubevirt.io/size: small + kubevirt.io/domain: "kojid-{{ meta.name }}-{{ koji_builder_type }}" + spec: + terminationGracePeriodSeconds: 30 + readinessProbe: + httpGet: + port: 5000 + path: /result.json + initialDelaySeconds: 120 + periodSeconds: 20 + timeoutSeconds: 10 + failureThreshold: 3 + successThreshold: 3 + domain: + resources: + requests: + memory: "{{ koji_builder_memory }}" + devices: + interfaces: + - name: default + masquerade: {} + disks: + - name: containerdisk + disk: + bus: virtio + - name: cloudinitdisk + disk: + bus: virtio + - name: mockdisk + serial: CVLY623300HK230D + disk: {} + - name: kojid + serial: CVLY623300HK240D + - name: client-cert + serial: CVLY623300HK250D + - name: ca-cert + serial: CVLY623300HK260D + - name: mockcfg + serial: CVLY623300HK270D + - name: cinitserver + serial: CVLY623300HK280D + networks: + - name: default + pod: {} + volumes: + - name: containerdisk + containerDisk: + image: "{{ koji_builder_image }}" + - name: cloudinitdisk + cloudInitConfigDrive: + secretRef: + name: "kojid-{{ meta.name }}-cloud-init" + - name: mockdisk + persistentVolumeClaim: + claimName: kojid-{{ meta.name }}-{{ koji_builder_type }}-mock + - name: kojid + configMap: + name: "{{ koji_builder_configmap }}" + - name: mockcfg + configMap: + name: kojid-mock-default + - name: cinitserver + configMap: + name: kojid-cloudinit-status-server + - name: client-cert + secret: + secretName: "{{ koji_builder_client_cert_secret }}" + - name: ca-cert + secret: + secretName: "{{ koji_builder_cacert_secret }}" + diff --git a/operator/roles/koji-builder/templates/kubevirt/vmi.yaml.j2 b/operator/roles/koji-builder/templates/kubevirt/vmi.yaml.j2 deleted file mode 100644 index b00b0ce..0000000 --- a/operator/roles/koji-builder/templates/kubevirt/vmi.yaml.j2 +++ /dev/null @@ -1,76 +0,0 @@ -apiVersion: kubevirt.io/v1alpha3 -kind: VirtualMachine -metadata: - name: "kojid-{{ meta.name }}-{{ koji_builder_type }}" - labels: - app: "kojid-{{ meta.name }}-{{ koji_builder_type }}" - namespace: {{ meta.namespace }} -spec: - running: {{ koji_builder_running }} - template: - metadata: - labels: - kubevirt.io/size: small - kubevirt.io/domain: "kojid-{{ meta.name }}-{{ koji_builder_type }}" - spec: - terminationGracePeriodSeconds: 30 - accessCredentials: - - sshPublicKey: - source: - secret: - secretName: "kojid-{{ meta.name }}-ssh-pub-keys" - propagationMethod: - configDrive: {} - domain: - resources: - requests: - memory: "{{ koji_builder_memory }}" - devices: - interfaces: - - name: default - masquerade: {} - disks: - - name: containerdisk - disk: - bus: virtio - - name: cloudinitdisk - disk: - bus: virtio - - name: mockdisk - serial: CVLY623300HK230D - disk: {} - - name: kojid - serial: CVLY623300HK240D - - name: client-cert - serial: CVLY623300HK250D - - name: ca-cert - serial: CVLY623300HK260D - - name: mockcfg - serial: CVLY623300HK270D - networks: - - name: default - pod: {} - volumes: - - name: containerdisk - containerDisk: - image: "{{ koji_builder_image }}" - - name: cloudinitdisk - cloudInitConfigDrive: - secretRef: - name: "kojid-{{ meta.name }}-cloud-init" - - name: mockdisk - persistentVolumeClaim: - claimName: kojid-{{ meta.name }}-{{ koji_builder_type }}-mock - - name: kojid - configMap: - name: "{{ koji_builder_configmap }}" - - name: mockcfg - configMap: - name: kojid-mock-default - - name: client-cert - secret: - secretName: "{{ koji_builder_client_cert_secret }}" - - name: ca-cert - secret: - secretName: "{{ koji_builder_cacert_secret }}" -