From 4c5b99f658d6e16f93c3d3c3184ed3a5b70fb94a Mon Sep 17 00:00:00 2001 From: Jan Ščotka Date: Feb 03 2025 09:51:20 +0000 Subject: [PATCH 1/6] fix npm deployment, remove versions of packages --- diff --git a/roles/openqa-cloud-webserver/files/Dockerfile b/roles/openqa-cloud-webserver/files/Dockerfile index 455995e..891ee6e 100644 --- a/roles/openqa-cloud-webserver/files/Dockerfile +++ b/roles/openqa-cloud-webserver/files/Dockerfile @@ -1,16 +1,7 @@ FROM quay.io/fedora/fedora LABEL title="openqa-webserver" -RUN dnf install -y openqa - -# openQA pull request #5463 "assets-via-npm" requires that we now get these packages locally -RUN dnf install -y nodejs && \ - npm install -g npm@10.5.0 && \ - cd /usr/share/openqa/ && \ - npm init -y && \ - npm install --save anser@^2.1.1 bootstrap@^4.6.1 chosen-js@^1.8.7 codemirror@^5.58.2 d3@^4.13.0 \ - dagre-d3@^0.6.4 datatables@^1.10.18 datatables.net-bs4@^1.13.8 fork-awesome@^1.2.0 jquery@^3.7.1 \ - jquery-ujs@^1.2.3 shepherd.js@^11.2.0 timeago@^1.6.7 +RUN dnf install -y openqa nodejs # do not force fixes as the specific version are required and will break otherwise RUN npm audit fix || true From 4ef1157fa725d925d8c210d114780b2f53176442 Mon Sep 17 00:00:00 2001 From: Jan Ščotka Date: Feb 03 2025 10:01:12 +0000 Subject: [PATCH 2/6] reduce number of workers on local machine in example to avoid resources exhaustion --- diff --git a/README.md b/README.md index cc42a49..d463476 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ Next, start some workers locally to run the tests. This example will start 16 wo ansible-playbook \ -e worker_ip_private=$(hostname -I | awk '{print $1}') \ -e use_http=true \ +-e number_of_workers=5 \ --ask-become-pass \ playbooks/hosts/openqa-cloud-worker-container.yml ``` From 43890f3aa8ba4c959a4fb0769961a6dde19ca8f8 Mon Sep 17 00:00:00 2001 From: Jan Ščotka Date: Feb 03 2025 12:09:56 +0000 Subject: [PATCH 3/6] new readme filefor centOS stream --- diff --git a/README.centos_stream.md b/README.centos_stream.md new file mode 100644 index 0000000..8b7d7b9 --- /dev/null +++ b/README.centos_stream.md @@ -0,0 +1,99 @@ +## About + +These part try to focus on Centos-stream testing effort within fedora like environment. +Documentation is more less same as main README.md + +#### Differences between fedora and centOS stream +We would prefer to use containerised setup also for workers. + - Dispatcher if service for fedora images, for centOS stream we have to create new to listen for new composes. + + +## Local cotainerised Deployment +to make it as close as possible to Fedora, there are some workarounds + +#### Start the openQA server +Start the openQA server containers using the Demo account, "Fake" authentication, and self-signed certificates: + +```bash +ansible-playbook \ +-e web_ip_private=$(hostname -I | awk '{print $1}') \ +-e auth=Fake \ +-e use_http=true \ +--ask-become-pass \ +playbooks/hosts/openqa-cloud-server.yml +``` +The first time it runs, it will take a while to build the podman images. Once the ansible playbook is finished, the containers will probably still be initializing; check on the containers with `podman ps -a` and `journalctl -f`. Keep checking `localhost` in your browser or just `curl localhost`. Make sure your browser will let you load a web page with self-signed certificates. + +#### Stop the fedora dispatcher +Dispatcher listen for Fedora images, so not necessary for CentOS. +```bash + sudo systemctl stop openqa-dispatcher + sudo systemctl disable openqa-dispatcher; +``` + +#### Load test templates +Once the server is up, click the login button on the user interface. Even though this example is using 'Fake' authentication, the test templates will be loaded using the Demo user's api key/secret. If the Demo user isn't logged in, the test templates won't load. Once the Demo user is logged in, load the test templates with this command: + +```bash +podman exec -it openqa-webserver /bin/bash -c "cd /var/lib/openqa/share/tests/fedora/; +./fifloader.py --load templates.fif.json templates-updates.fif.json;" +``` +The successfully loaded templates will appear in the *Job Groups* drop-down menu. + +#### load CentOS image +Currently there is ised `DISTRI=fedora` as workaround, to be able trigger tests for fedora. +Otherwise there is possible to add new product and use it then (http://localhost:8080/admin/products) +```bash +ISONAME=CentOS-stream-1.iso +curl -o ~/openqa-webserver/iso/$ISONAME https://composes.stream.centos.org/stream-10/production/latest-CentOS-Stream/compose/BaseOS/x86_64/iso/CentOS-Stream-10-20250203.0-x86_64-dvd1.iso +podman exec -it openqa-webserver bash -c " + cd /var/lib/openqa/share/factory/iso; + openqa-cli api -X POST isos \ + ISO=$ISONAME \ + DISTRI=fedora \ + VERSION=10 \ + FLAVOR=Everything-boot-iso \ + ARCH=x86_64 \ + BUILD=IDENTIFIER_OF_COMPOSE +" +``` + +#### Start qemu localhost workers in container +Next, start some workers locally to run the tests. This example will start 5 worker containers along with a corresponding systemd service for each worker container. Make sure that the Demo user is still logged in because the workers will also use the Demo user's api key/secret. If the Demo user isn't logged in, the workers will shut themselves down. + +```bash +ansible-playbook \ +-e worker_ip_private=$(hostname -I | awk '{print $1}') \ +-e use_http=true \ +-e number_of_workers=5 \ +--ask-become-pass \ +playbooks/hosts/openqa-cloud-worker-container.yml +``` + +See a list of all the containerized workers: +```bash +sudo systemctl list-units --all 'openqa-worker@*.service'; +``` +In addition to the workers, there will be a new systemd service running on the host machine: + - **openqa-createhdds**: a service that runs [createhdds](https://pagure.io/fedora-qa/createhdds) to build images that can't be fetched remotely. + +If you have limited disk space or can't wait several hours for the openqa-createhdds service to finish generating local images, then just stop and disable the openqa-createhdds service: +`sudo systemctl stop open-createhdds && sudo systemctl disable openqa-createhdds` + +Click through the web user interface to monitor the test progress. Initially, some of the tests will fail while waiting for the openqa-createhdds service to finish generating local images. This process can take several hours. Also, tests that require tap devices, won't run in containers; use the playbook `openqa-cloud-worker.yml` instead for tap tests. + + +#### STOP webserver and workers + +Stop the web server and components manually. Make sure to stop the database last or it won't exit gracefully. + +```bash +for service in openqa-test-update.timer openqa-test-update openqa-createhdds.timer openqa-createhdds openqa-reverse-proxy openqa-webserver openqa-database; do + sudo systemctl stop $service && sudo systemctl disable $service; +done +``` + +To stop containerized workers: +```bash +sudo systemctl stop openqa-worker@{0..4} && sudo systemctl disable openqa-worker@{0..4} +``` \ No newline at end of file From 736a8c0c6fa80e6ead3be614a1cac076b978ac36 Mon Sep 17 00:00:00 2001 From: Jan Ščotka Date: Feb 14 2025 10:14:20 +0000 Subject: [PATCH 4/6] add ansible cleanup script for containerised setup --- diff --git a/README.centos_stream.md b/README.centos_stream.md index 8b7d7b9..abb7f04 100644 --- a/README.centos_stream.md +++ b/README.centos_stream.md @@ -1,18 +1,21 @@ ## About -These part try to focus on Centos-stream testing effort within fedora like environment. -Documentation is more less same as main README.md +This part focus on `CentOS stream` testing effort within fedora like environment. +Documentation is very similar as main README.md, there are few mentioned differences #### Differences between fedora and centOS stream We would prefer to use containerised setup also for workers. - - Dispatcher if service for fedora images, for centOS stream we have to create new to listen for new composes. + +- Dispatcher will be disabled - dispatcher is looking for fedora images, not intended for centOS stream we have to create new to listen for new composes. +- decreased number of workers + ## Local cotainerised Deployment to make it as close as possible to Fedora, there are some workarounds -#### Start the openQA server -Start the openQA server containers using the Demo account, "Fake" authentication, and self-signed certificates: +#### Start the openQA server and Stop the fedora dispatcher +Start the openQA server containers using the Demo account, "Fake" authentication, and self-signed certificates. Dispatcher listen for Fedora images, so not necessary for CentOS. ```bash ansible-playbook \ @@ -21,18 +24,18 @@ ansible-playbook \ -e use_http=true \ --ask-become-pass \ playbooks/hosts/openqa-cloud-server.yml + + sudo systemctl stop openqa-dispatcher + sudo systemctl disable openqa-dispatcher ``` The first time it runs, it will take a while to build the podman images. Once the ansible playbook is finished, the containers will probably still be initializing; check on the containers with `podman ps -a` and `journalctl -f`. Keep checking `localhost` in your browser or just `curl localhost`. Make sure your browser will let you load a web page with self-signed certificates. -#### Stop the fedora dispatcher -Dispatcher listen for Fedora images, so not necessary for CentOS. -```bash - sudo systemctl stop openqa-dispatcher - sudo systemctl disable openqa-dispatcher; -``` +#### Login into server +Once the server is up, click the login button on the user interface. Even though this example is using 'Fake' authentication, the test templates will be loaded using the Demo user's api key/secret. If the Demo user isn't logged in, the test templates won't load. #### Load test templates -Once the server is up, click the login button on the user interface. Even though this example is using 'Fake' authentication, the test templates will be loaded using the Demo user's api key/secret. If the Demo user isn't logged in, the test templates won't load. Once the Demo user is logged in, load the test templates with this command: +Templates defines products (distros) and tests matrix. + Once the Demo user is logged in, load the test templates with this command: ```bash podman exec -it openqa-webserver /bin/bash -c "cd /var/lib/openqa/share/tests/fedora/; @@ -41,11 +44,12 @@ podman exec -it openqa-webserver /bin/bash -c "cd /var/lib/openqa/share/tests/fe The successfully loaded templates will appear in the *Job Groups* drop-down menu. #### load CentOS image -Currently there is ised `DISTRI=fedora` as workaround, to be able trigger tests for fedora. +Currently there is used `DISTRI=fedora` as workaround, to be able trigger tests for fedora. Otherwise there is possible to add new product and use it then (http://localhost:8080/admin/products) ```bash ISONAME=CentOS-stream-1.iso -curl -o ~/openqa-webserver/iso/$ISONAME https://composes.stream.centos.org/stream-10/production/latest-CentOS-Stream/compose/BaseOS/x86_64/iso/CentOS-Stream-10-20250203.0-x86_64-dvd1.iso +BUILD_ID=12345 +curl -o ~/openqa-webserver/iso/$ISONAME https://composes.stream.centos.org/stream-10/production/latest-CentOS-Stream/compose/BaseOS/x86_64/iso/CentOS-Stream-10-20250214.0-x86_64-dvd1.iso podman exec -it openqa-webserver bash -c " cd /var/lib/openqa/share/factory/iso; openqa-cli api -X POST isos \ @@ -54,7 +58,7 @@ podman exec -it openqa-webserver bash -c " VERSION=10 \ FLAVOR=Everything-boot-iso \ ARCH=x86_64 \ - BUILD=IDENTIFIER_OF_COMPOSE + BUILD=$BUILD_ID " ``` @@ -84,6 +88,10 @@ Click through the web user interface to monitor the test progress. Initially, so #### STOP webserver and workers +There is automated ansible playbook, to cleanup env and stored data: +```bash +ansible-playbook --ask-become-pass playbooks/hosts/openqa-cloud-container-cleanup.yml +``` Stop the web server and components manually. Make sure to stop the database last or it won't exit gracefully. diff --git a/playbooks/hosts/openqa-cloud-container-cleanup.yml b/playbooks/hosts/openqa-cloud-container-cleanup.yml new file mode 100644 index 0000000..d942578 --- /dev/null +++ b/playbooks/hosts/openqa-cloud-container-cleanup.yml @@ -0,0 +1,68 @@ +- name: OpenQA_Cloud_cleanup + hosts: openqa_cloud_server + vars: + openqa_cloud_dir: "{{ dir | default(ansible_env.HOME) }}" + openqa_database_dir: "{{ openqa_cloud_dir }}/openqa-database" + openqa_webserver_dir: "{{ openqa_cloud_dir }}/openqa-webserver" + openqa_dispatcher_dir: "{{ openqa_cloud_dir }}/openqa-dispatcher" + openqa_worker_dir: "{{ openqa_cloud_dir }}/openqa-worker" + openqa_reverse_proxy_dir: "{{ openqa_cloud_dir }}/openqa-reverse-proxy" + + tasks: + - name: Stop and disable systemd services for OpenQA server containers + become: true + ansible.builtin.systemd_service: + name: "{{ item }}" + state: stopped + enabled: false + loop: + - openqa-test-update.timer + - openqa-test-update + - openqa-createhdds.timer + - openqa-createhdds + - openqa-reverse-proxy + - openqa-dispatcher + - openqa-webserver + - openqa-database + + - name: Stop and disable systemd services for OpenQA worker containers + become: true + ansible.builtin.systemd_service: + name: "{{ item }}" + state: stopped + enabled: false + loop: + - openqa-worker@0 + - openqa-worker@1 + - openqa-worker@2 + - openqa-worker@3 + - openqa-worker@4 + + - name: Remove OpenQA service dirs + become: true + ansible.builtin.file: + state: absent + path: "{{ item }}" + loop: + - "{{ openqa_database_dir }}" + - "{{ openqa_webserver_dir }}" + - "{{ openqa_dispatcher_dir }}" + - "{{ openqa_worker_dir }}" + - "{{ openqa_reverse_proxy_dir }}" + + - name: Remove podman images + containers.podman.podman_image: + name: "{{ item }}" + state: absent + force: true + loop: + - localhost/openqa-worker + - localhost/openqa-dispatcher + - localhost/openqa-webserver + - localhost/openqa-database + + handlers: + - name: Reload systemd + become: true + ansible.builtin.systemd_service: + daemon_reload: true From e58ea27cb6a56dc2956344e2be9079ce1461a708 Mon Sep 17 00:00:00 2001 From: Jan Ščotka Date: Mar 13 2025 13:17:22 +0000 Subject: [PATCH 5/6] several changes --- diff --git a/README.centos_stream.md b/README.centos_stream.md index abb7f04..0793cc8 100644 --- a/README.centos_stream.md +++ b/README.centos_stream.md @@ -39,7 +39,7 @@ Templates defines products (distros) and tests matrix. ```bash podman exec -it openqa-webserver /bin/bash -c "cd /var/lib/openqa/share/tests/fedora/; -./fifloader.py --load templates.fif.json templates-updates.fif.json;" +./fifloader.py --load --no-validate templates/centos-stream.json" ``` The successfully loaded templates will appear in the *Job Groups* drop-down menu. @@ -47,16 +47,18 @@ The successfully loaded templates will appear in the *Job Groups* drop-down menu Currently there is used `DISTRI=fedora` as workaround, to be able trigger tests for fedora. Otherwise there is possible to add new product and use it then (http://localhost:8080/admin/products) ```bash -ISONAME=CentOS-stream-1.iso +URL=https://composes.stream.centos.org/stream-10/production/latest-CentOS-Stream/compose/BaseOS/x86_64/iso/ BUILD_ID=12345 -curl -o ~/openqa-webserver/iso/$ISONAME https://composes.stream.centos.org/stream-10/production/latest-CentOS-Stream/compose/BaseOS/x86_64/iso/CentOS-Stream-10-20250214.0-x86_64-dvd1.iso +ISONAME=CentOS-Stream-x86_64-dvd-iso-$BUILD_ID.iso + +curl -o ~/openqa-webserver/iso/$ISONAME https://composes.stream.centos.org/stream-10/production/latest-CentOS-Stream/compose/BaseOS/x86_64/iso/CentOS-Stream-10-20250312.0-x86_64-dvd1.iso podman exec -it openqa-webserver bash -c " cd /var/lib/openqa/share/factory/iso; openqa-cli api -X POST isos \ ISO=$ISONAME \ - DISTRI=fedora \ + DISTRI=centos \ VERSION=10 \ - FLAVOR=Everything-boot-iso \ + FLAVOR=BaseOS-dvd-iso \ ARCH=x86_64 \ BUILD=$BUILD_ID " From 4ada497856166b1d3415f3b0ea4f75dc0a858c07 Mon Sep 17 00:00:00 2001 From: Jan Ščotka Date: Mar 27 2025 11:08:31 +0000 Subject: [PATCH 6/6] fix items for centos --- diff --git a/README.centos_stream.md b/README.centos_stream.md index 0793cc8..074db61 100644 --- a/README.centos_stream.md +++ b/README.centos_stream.md @@ -48,10 +48,10 @@ Currently there is used `DISTRI=fedora` as workaround, to be able trigger tests Otherwise there is possible to add new product and use it then (http://localhost:8080/admin/products) ```bash URL=https://composes.stream.centos.org/stream-10/production/latest-CentOS-Stream/compose/BaseOS/x86_64/iso/ -BUILD_ID=12345 +BUILD_ID=20250324.0 ISONAME=CentOS-Stream-x86_64-dvd-iso-$BUILD_ID.iso -curl -o ~/openqa-webserver/iso/$ISONAME https://composes.stream.centos.org/stream-10/production/latest-CentOS-Stream/compose/BaseOS/x86_64/iso/CentOS-Stream-10-20250312.0-x86_64-dvd1.iso +curl -o ~/openqa-webserver/iso/$ISONAME https://composes.stream.centos.org/stream-10/production/latest-CentOS-Stream/compose/BaseOS/x86_64/iso/CentOS-Stream-10-20250324.0-x86_64-dvd1.iso podman exec -it openqa-webserver bash -c " cd /var/lib/openqa/share/factory/iso; openqa-cli api -X POST isos \ diff --git a/roles/openqa-cloud-webserver/tasks/main.yml b/roles/openqa-cloud-webserver/tasks/main.yml index f8bf4d1..b269645 100644 --- a/roles/openqa-cloud-webserver/tasks/main.yml +++ b/roles/openqa-cloud-webserver/tasks/main.yml @@ -27,7 +27,7 @@ repo: 'https://pagure.io/fedora-qa/os-autoinst-distri-fedora' dest: "{{ openqa_webserver_dir }}/tests/fedora" single_branch: true - depth: 5 + depth: 1 - name: Copy Dockerfile to openqa-webserver ansible.builtin.copy: diff --git a/roles/openqa-cloud-worker-container/tasks/main.yml b/roles/openqa-cloud-worker-container/tasks/main.yml index c84208a..a0da4b9 100644 --- a/roles/openqa-cloud-worker-container/tasks/main.yml +++ b/roles/openqa-cloud-worker-container/tasks/main.yml @@ -29,12 +29,13 @@ cmd: git config --global --add safe.directory {{ test_directory }} when: test_directory not in git_safe_directories.stdout_lines -- name: Git checkout fedora tests and needles - ansible.builtin.git: - repo: 'https://pagure.io/fedora-qa/os-autoinst-distri-fedora' - dest: "{{ test_directory }}" - single_branch: true - depth: 5 +# already done by web server in case done on one machine +#- name: Git checkout fedora tests and needles +# ansible.builtin.git: +# repo: 'https://pagure.io/fedora-qa/os-autoinst-distri-fedora' +# dest: "{{ test_directory }}" +# single_branch: true +# depth: 1 - name: Copy Dockerfile to openqa-worker ansible.builtin.copy: diff --git a/roles/openqa-cloud-worker-container/templates/start-openqa-worker.sh.j2 b/roles/openqa-cloud-worker-container/templates/start-openqa-worker.sh.j2 index c627825..30d8b79 100755 --- a/roles/openqa-cloud-worker-container/templates/start-openqa-worker.sh.j2 +++ b/roles/openqa-cloud-worker-container/templates/start-openqa-worker.sh.j2 @@ -18,6 +18,7 @@ VNC_PORT=$((OPENQA_WORKER_INSTANCE + 5990)) -p $VNC_PORT:$VNC_PORT \ -v {{ openqa_worker_dir }}/conf:/conf:z \ -v {{ openqa_worker_dir }}/tests/fedora:/var/lib/openqa/share/tests/fedora:z \ + -v {{ openqa_worker_dir }}/tests/centos:/var/lib/openqa/share/tests/centos:z \ -v {{ openqa_worker_dir }}/init-openqa-worker.sh:/init-openqa-worker.sh:z \ -v {{ openqa_worker_dir }}/{{ openqa_hostname_cloud }}:/var/lib/openqa/cache/{{ openqa_hostname_cloud }}:z \ localhost/openqa-worker:latest /init-openqa-worker.sh