From cc94fca0061c9b38d56d9964c98a743e9bf29b26 Mon Sep 17 00:00:00 2001
From: Fabien Boucher <fboucher@redhat.com>
Date: Feb 05 2020 16:19:52 +0000
Subject: [PATCH 1/6] Add the source-to-tarball role


This role is aimed to be used with a source that contain a RPM spec file.
It detects the version of the source from the git repository and it creates
the tarball from the source. Then it updates the spec file
to change the Source0 and Version fields.

The version detection could be optionally based on detecting the version
from the spec file.

---

diff --git a/roles/source-to-tarball/defaults/main.yaml b/roles/source-to-tarball/defaults/main.yaml
new file mode 100644
index 0000000..0dd6a65
--- /dev/null
+++ b/roles/source-to-tarball/defaults/main.yaml
@@ -0,0 +1 @@
+version_from_specfile: false
diff --git a/roles/source-to-tarball/files/detect-source-version.py b/roles/source-to-tarball/files/detect-source-version.py
new file mode 100755
index 0000000..2f6794a
--- /dev/null
+++ b/roles/source-to-tarball/files/detect-source-version.py
@@ -0,0 +1,42 @@
+#!/bin/env python
+
+import os
+import sys
+import subprocess
+
+
+def execute(argv, capture=False, cwd=None, test=False):
+    if capture is True:
+        stdout = subprocess.PIPE
+    else:
+        stdout = None
+    if os.environ.get('debug'):
+        print("Running %s" % argv)
+    s = subprocess.Popen(argv, stdout=stdout, cwd=cwd)
+    out, _ = s.communicate()
+    code = s.wait()
+    if os.environ.get('debug'):
+        print("Command %s exited with code %s" % (argv, code))
+    if code:
+        if not test:
+            print("Command %s failed" % argv)
+        raise RuntimeError()
+    return out
+
+
+def get_repo_version(repo):
+    ref = 'HEAD'
+    try:
+        version = execute(["git", "describe", "--tags", ref],
+                          capture=True, cwd=repo, test=True).strip()
+    except RuntimeError:
+        version = execute(["git", "rev-list", "--count", ref],
+                          capture=True, cwd=repo, test=True).strip()
+        # Make sure rev-list version is lower than first tag
+        version = "0.0.0.0-dev%s" % version
+    return version.replace('-', '.')
+
+
+if __name__ == "__main__":
+    if os.path.isdir(os.path.join(sys.argv[1], '.git')):
+        print(get_repo_version(sys.argv[1]))
diff --git a/roles/source-to-tarball/tasks/main.yaml b/roles/source-to-tarball/tasks/main.yaml
new file mode 100644
index 0000000..4aae42e
--- /dev/null
+++ b/roles/source-to-tarball/tasks/main.yaml
@@ -0,0 +1,67 @@
+---
+- name: Discover package name from specfile name
+  shell: ls *.spec | sed 's/\.spec$//'
+  args:
+    chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
+  register: get_package_name
+
+- name: Set package name fact
+  set_fact:
+    package_name: "{{ get_package_name.stdout }}"
+
+- name: Set tarball path fact
+  set_fact:
+    tarball: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/{{ package_name }}-HEAD.tgz"
+
+- block:
+  - name: Detect version from specfile
+    shell: >
+      rpmspec -P {{ package_name}}.spec | egrep -i '^version:' |
+      sed -e "s/[Vv]ersion:[ \t]\+//g"
+    register: cmd
+    args:
+      chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
+
+  - name: Set source_version fact
+    set_fact:
+      source_version: "{{ cmd.stdout|trim }}"
+
+  when: version_from_specfile
+
+- block:
+  - name: Detect source version from git
+    script:
+       cmd: >
+         files/detect-source-version.py
+         {{ ansible_user_dir }}/{{ zuul.project.src_dir }}
+    register: cmd
+
+  - name: Set source_version fact
+    set_fact:
+      source_version: "{{ cmd.stdout|trim }}"
+
+  when: not version_from_specfile
+
+- name: Create the source tarball
+  command: >
+    tar -czf /tmp/HEAD.tgz
+    --exclude .git
+    --transform s/^{{ package_name }}/{{ package_name }}-{{ source_version }}/
+    {{ zuul.project.src_dir|basename }}
+  args:
+    chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/.."
+
+- name: Move tarball in the project repository
+  command: mv /tmp/HEAD.tgz {{ tarball }}
+
+- name: Set specfile Version
+  replace:
+    path: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/{{ package_name }}.spec"
+    regexp: "^[Vv]ersion:.*"
+    replace: "Version: {{ source_version }}"
+
+- name: Set specfile Source0
+  replace:
+    path: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/{{ package_name }}.spec"
+    regexp: "^[Ss]ource0:.*"
+    replace: "Source0: {{ tarball | basename }}"

From 1dce28af9f9fd896819a4a807a0100b1fd558bd5 Mon Sep 17 00:00:00 2001
From: Fabien Boucher <fboucher@redhat.com>
Date: Feb 06 2020 09:03:44 +0000
Subject: [PATCH 2/6] mock-build: use -a to set intermediary repos instead of copyin


---

diff --git a/roles/mock-build/tasks/main.yaml b/roles/mock-build/tasks/main.yaml
index 90bded5..d92ba2a 100644
--- a/roles/mock-build/tasks/main.yaml
+++ b/roles/mock-build/tasks/main.yaml
@@ -1,33 +1,32 @@
 ---
 - name: Set mock options fact
   set_fact:
-    mock_opts: "-v -r {{ mock_config }} --resultdir {{ ansible_user_dir }}/repo"
+    mock_opts: "-v -r {{ mock_config }} --localrepo {{ ansible_user_dir }}/repo"
 
 - name: Init the mock chroot
   command: mock {{ mock_opts }} --init
 
-- name: Create intermediary repositories yum repos file
-  blockinfile:
-    path: "{{ ansible_user_dir }}/artifacts.repo"
-    create: true
-    content: |
-      {% for artifact in zuul.artifacts|default([]) %}{% if artifact.name == 'repo' %}
-      [{{ artifact.project | regex_replace('/', '_') }}-{{ artifact.change }}-{{ artifact.patchset }}]
-      name={{ artifact.project }}-{{ artifact.change }}-{{ artifact.patchset }}
-      enabled=1
-      baseurl={{ artifact.url }}
-      gpgcheck=0
-
+- name: Create intermediary repos baseurls for mock
+  set_fact:
+    baseurls: >
+      {% for artifact in zuul.artifacts|default([]) %}
+      {% if artifact.name == 'repo' %} -a {{ artifact.url }}
       {% endif %}{% endfor %}
 
-- name: Copy artifacts.repo in the mock root
+- name: Build the RPM
   shell: >
-    mock {{ mock_opts }}
-    --copyin {{ ansible_user_dir }}/artifacts.repo /etc/yum.repos.d/
+    mock {{ mock_opts }} {{ baseurls|trim }} --postinstall --chain {{ srpm }}
 
-- name: Build the RPM
+- name: Copy results
   shell: >
-    mock {{ mock_opts }} --postinstall --rebuild {{ srpm }}
+    find {{ ansible_user_dir }}/repo/results/ -name {{ item }}
+    -exec cp {} {{ ansible_user_dir }}/repo/ \;
+  loop:
+    - "*.rpm"
+    - "*.log"
+
+- name: Remove results directory created by mock
+  command: rm -Rf {{ ansible_user_dir }}/repo/results
 
 - name: Create repository
   command: createrepo .

From 7008f75dbaf5737bae9acd9c8b4372b1aa716b10 Mon Sep 17 00:00:00 2001
From: Fabien Boucher <fboucher@redhat.com>
Date: Feb 12 2020 08:42:56 +0000
Subject: [PATCH 3/6] Add mock-build-from-source job


This job is aimed to be used with source projects that host a
specfile into their git repository.

The job also handles the build of required-projects to make them
available in next build mock build root.

---

diff --git a/playbooks/rpm/mock-build-source.yaml b/playbooks/rpm/mock-build-source.yaml
new file mode 100644
index 0000000..4a7ed88
--- /dev/null
+++ b/playbooks/rpm/mock-build-source.yaml
@@ -0,0 +1,31 @@
+- hosts: all
+  vars:
+  tasks:
+    - set_fact:
+        project_deps_to_build: "{{ zuul.projects.values() | list | json_query('[?required]') }}"
+        project_to_build:
+          - "{{ zuul.project }}"
+        srpms: []
+
+    - include_role:
+        name: ensure-mock
+
+    - include_role:
+        name: source-to-tarball
+      vars:
+        zuul:
+          project: "{{ item }}"
+      loop: "{{ project_deps_to_build | union(project_to_build) }}"
+
+    - include_role:
+        name: mock-srpm-build
+      vars:
+        zuul:
+          project: "{{ item }}"
+      loop: "{{ project_deps_to_build | union(project_to_build) }}"
+
+    - include_role:
+        name: mock-build
+      vars:
+        srpm: "{{ item }}"
+      loop: "{{ srpms }}"
diff --git a/roles/mock-build/tasks/main.yaml b/roles/mock-build/tasks/main.yaml
index d92ba2a..46eff73 100644
--- a/roles/mock-build/tasks/main.yaml
+++ b/roles/mock-build/tasks/main.yaml
@@ -1,34 +1,24 @@
 ---
 - name: Set mock options fact
   set_fact:
-    mock_opts: "-v -r {{ mock_config }} --localrepo {{ ansible_user_dir }}/repo"
+    mock_opts: "-v -r {{ mock_config }} --localrepo {{ ansible_user_dir }}/chainrepo"
 
 - name: Init the mock chroot
   command: mock {{ mock_opts }} --init
 
 - name: Create intermediary repos baseurls for mock
   set_fact:
-    baseurls: >
+    artifacts_baseurls: >
       {% for artifact in zuul.artifacts|default([]) %}
       {% if artifact.name == 'repo' %} -a {{ artifact.url }}
       {% endif %}{% endfor %}
 
 - name: Build the RPM
   shell: >
-    mock {{ mock_opts }} {{ baseurls|trim }} --postinstall --chain {{ srpm }}
+    mock {{ mock_opts }} {{ artifacts_baseurls|trim }} --postinstall
+    --chain {{ srpm }}
 
-- name: Copy results
-  shell: >
-    find {{ ansible_user_dir }}/repo/results/ -name {{ item }}
-    -exec cp {} {{ ansible_user_dir }}/repo/ \;
-  loop:
-    - "*.rpm"
-    - "*.log"
-
-- name: Remove results directory created by mock
-  command: rm -Rf {{ ansible_user_dir }}/repo/results
-
-- name: Create repository
-  command: createrepo .
-  args:
-    chdir: "{{ ansible_user_dir }}/repo"
+- name: Sync chained repo in {{ ansible_user_dir }}/repo
+  command: >
+    rsync -av {{ ansible_user_dir }}/chainrepo/results/{{ mock_config }}/
+    {{ ansible_user_dir }}/repo/
diff --git a/roles/mock-srpm-build/tasks/main.yaml b/roles/mock-srpm-build/tasks/main.yaml
index 49688f3..ac9af53 100644
--- a/roles/mock-srpm-build/tasks/main.yaml
+++ b/roles/mock-srpm-build/tasks/main.yaml
@@ -1,4 +1,14 @@
 ---
+- name: Discover package name from specfile name
+  shell: ls *.spec | sed 's/\.spec$//'
+  args:
+    chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
+  register: get_package_name
+
+- name: Set package name fact
+  set_fact:
+    package_name: "{{ get_package_name.stdout }}"
+
 - name: Set mock options fact
   set_fact:
     mock_opts: "-v -r {{ mock_config }} --resultdir {{ ansible_user_dir }}/repo"
@@ -20,7 +30,7 @@
 - name: Rename log file with srpm prefix
   command: |
     mv {{ ansible_user_dir }}/repo/{{ item }}
-    {{ ansible_user_dir }}/repo/srpm_{{ item }}
+    {{ ansible_user_dir }}/repo/srpm_{{ package_name }}_{{ item }}
   loop:
     - build.log
     - hw_info.log
@@ -28,9 +38,13 @@
     - state.log
 
 - name: Get SRPM path
-  shell: ls {{ ansible_user_dir }}/repo/*.src.rpm
+  shell: ls {{ ansible_user_dir }}/repo/{{ package_name }}-*.src.rpm
   register: result
 
 - name: Set SRPM path fact
   set_fact:
     srpm: "{{ result.stdout }}"
+
+- name: Set SRPMs path fact
+  set_fact:
+    srpms: "{{ (srpms | default([])) | union([result.stdout]) }}"
diff --git a/roles/source-to-tarball/files/detect-source-version.py b/roles/source-to-tarball/files/detect-source-version.py
index 2f6794a..8014647 100755
--- a/roles/source-to-tarball/files/detect-source-version.py
+++ b/roles/source-to-tarball/files/detect-source-version.py
@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/usr/bin/env python3
 
 import os
 import sys
diff --git a/roles/source-to-tarball/tasks/main.yaml b/roles/source-to-tarball/tasks/main.yaml
index 4aae42e..51a2bbc 100644
--- a/roles/source-to-tarball/tasks/main.yaml
+++ b/roles/source-to-tarball/tasks/main.yaml
@@ -16,7 +16,7 @@
 - block:
   - name: Detect version from specfile
     shell: >
-      rpmspec -P {{ package_name}}.spec | egrep -i '^version:' |
+      rpmspec -P {{ package_name}}.spec 2> /dev/null | egrep -i '^version:' |
       sed -e "s/[Vv]ersion:[ \t]\+//g"
     register: cmd
     args:
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index 6b3e230..87a005d 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -17,6 +17,30 @@
     post-run: playbooks/rpm/repo-fetch.yaml
 
 - job:
+    name: rpm-mock-build-source
+    description: |
+      Experimental jobs that build rpm from source where
+      a specfile is included. It also build required-projects
+
+      Responds to these variables:
+
+      .. zuul:jobvar:: mock_config
+         : default: fedora-rawhide-x86_64
+
+         Mock config from /etc/mock/.
+
+      .. zuul:jobvar:: version_from_specfile
+         : default: False
+
+         Using or not the version detected from
+         the specfile.
+
+    provides: repo
+    requires: repo
+    run: playbooks/rpm/mock-build-source.yaml
+    post-run: playbooks/rpm/repo-fetch.yaml
+
+- job:
     name: rpm-lint
     description: |
       Base job to run rpm-lint on rpms

From 9b392ba58a4adb0b9fa861dadf7f7044c80afe98 Mon Sep 17 00:00:00 2001
From: Fabien Boucher <fboucher@redhat.com>
Date: Feb 13 2020 15:12:56 +0000
Subject: [PATCH 4/6] rpm-mock-build-source: filter artifacts from to_build_deps


---

diff --git a/playbooks/rpm/mock-build-source.yaml b/playbooks/rpm/mock-build-source.yaml
index 4a7ed88..fcb58fe 100644
--- a/playbooks/rpm/mock-build-source.yaml
+++ b/playbooks/rpm/mock-build-source.yaml
@@ -11,6 +11,9 @@
         name: ensure-mock
 
     - include_role:
+        name: filter-artifacts
+
+    - include_role:
         name: source-to-tarball
       vars:
         zuul:
diff --git a/roles/filter-artifacts/files/filter.py b/roles/filter-artifacts/files/filter.py
new file mode 100755
index 0000000..19c3235
--- /dev/null
+++ b/roles/filter-artifacts/files/filter.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+import sys
+import json
+
+artifacts = json.loads(sys.argv[1])
+deps_to_build = json.loads(sys.argv[2])
+
+artifacts = [artifact['project'] for artifact in artifacts]
+
+deps_to_build = list(filter(
+    lambda x: x['name'] not in artifacts, deps_to_build))
+
+print(json.dumps(deps_to_build))
diff --git a/roles/filter-artifacts/tasks/main.yml b/roles/filter-artifacts/tasks/main.yml
new file mode 100644
index 0000000..e286be6
--- /dev/null
+++ b/roles/filter-artifacts/tasks/main.yml
@@ -0,0 +1,10 @@
+- name: Remove project that exposed artifact from build list
+  script:
+    cmd: >
+      filter.py '{{ zuul.artifacts | to_json }}'
+      '{{ project_deps_to_build | to_json }}'
+  register: _filter
+
+- name: Store back the result in project_deps_to_build fact
+  set_fact:
+    project_deps_to_build: "{{ _filter.stdout | trim }}"

From 615aa169e7fd7998a7726ce177d1897ffb8c45bd Mon Sep 17 00:00:00 2001
From: Fabien Boucher <fboucher@redhat.com>
Date: Feb 13 2020 15:48:13 +0000
Subject: [PATCH 5/6] unify rpm-mock-build with rpm-mock-build-source


---

diff --git a/playbooks/rpm/mock-build-source.yaml b/playbooks/rpm/mock-build-source.yaml
deleted file mode 100644
index fcb58fe..0000000
--- a/playbooks/rpm/mock-build-source.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-- hosts: all
-  vars:
-  tasks:
-    - set_fact:
-        project_deps_to_build: "{{ zuul.projects.values() | list | json_query('[?required]') }}"
-        project_to_build:
-          - "{{ zuul.project }}"
-        srpms: []
-
-    - include_role:
-        name: ensure-mock
-
-    - include_role:
-        name: filter-artifacts
-
-    - include_role:
-        name: source-to-tarball
-      vars:
-        zuul:
-          project: "{{ item }}"
-      loop: "{{ project_deps_to_build | union(project_to_build) }}"
-
-    - include_role:
-        name: mock-srpm-build
-      vars:
-        zuul:
-          project: "{{ item }}"
-      loop: "{{ project_deps_to_build | union(project_to_build) }}"
-
-    - include_role:
-        name: mock-build
-      vars:
-        srpm: "{{ item }}"
-      loop: "{{ srpms }}"
diff --git a/playbooks/rpm/mock-build.yaml b/playbooks/rpm/mock-build.yaml
index 5a77b5a..83c4852 100644
--- a/playbooks/rpm/mock-build.yaml
+++ b/playbooks/rpm/mock-build.yaml
@@ -1,5 +1,35 @@
 - hosts: all
-  roles:
-    - ensure-mock
-    - mock-srpm-build
-    - mock-build
+  vars:
+    project_deps_to_build: "{{ zuul.projects.values() | list | json_query('[?required]') }}"
+    project_to_build:
+      - "{{ zuul.project }}"
+    build_from_source: "{{ build_from_source | default(false) }}"
+    version_from_specfile: "{{ version_from_specfile | default(false) }}"
+    srpms: []
+  tasks:
+    - include_role:
+        name: ensure-mock
+
+    - include_role:
+        name: filter-artifacts
+
+    - include_role:
+        name: source-to-tarball
+      vars:
+        zuul:
+          project: "{{ item }}"
+      loop: "{{ project_deps_to_build | union(project_to_build) }}"
+      when: build_from_source
+
+    - include_role:
+        name: mock-srpm-build
+      vars:
+        zuul:
+          project: "{{ item }}"
+      loop: "{{ project_deps_to_build | union(project_to_build) }}"
+
+    - include_role:
+        name: mock-build
+      vars:
+        srpm: "{{ item }}"
+      loop: "{{ srpms }}"
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index 87a005d..cd53df1 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -11,23 +11,12 @@
 
          Mock config from /etc/mock/.
 
-    provides: repo
-    requires: repo
-    run: playbooks/rpm/mock-build.yaml
-    post-run: playbooks/rpm/repo-fetch.yaml
-
-- job:
-    name: rpm-mock-build-source
-    description: |
-      Experimental jobs that build rpm from source where
-      a specfile is included. It also build required-projects
-
-      Responds to these variables:
-
-      .. zuul:jobvar:: mock_config
-         : default: fedora-rawhide-x86_64
+      .. zuul:jobvar:: build_from_source
+         : default: False
 
-         Mock config from /etc/mock/.
+         Set it to true if specfile is stored in
+         the same repository than the source. The
+         source will be used to create the source tarball.
 
       .. zuul:jobvar:: version_from_specfile
          : default: False
@@ -37,7 +26,7 @@
 
     provides: repo
     requires: repo
-    run: playbooks/rpm/mock-build-source.yaml
+    run: playbooks/rpm/mock-build.yaml
     post-run: playbooks/rpm/repo-fetch.yaml
 
 - job:

From 2efd2b5dfd769c484ab9f2fb156d1d1a81ecaba6 Mon Sep 17 00:00:00 2001
From: Fabien Boucher <fboucher@redhat.com>
Date: Feb 13 2020 16:00:42 +0000
Subject: [PATCH 6/6] rpm-mock-build: handle case no zuul.artifacts


---

diff --git a/playbooks/rpm/mock-build-source.yaml b/playbooks/rpm/mock-build-source.yaml
new file mode 100644
index 0000000..681843e
--- /dev/null
+++ b/playbooks/rpm/mock-build-source.yaml
@@ -0,0 +1,36 @@
+- hosts: all
+  vars:
+    project_deps_to_build: "{{ zuul.projects.values() | list | json_query('[?required]') }}"
+    project_to_build:
+      - "{{ zuul.project }}"
+    build_from_source: "{{ build_from_source | default(false) }}"
+    version_from_specfile: "{{ version_from_specfile | default(false) }}"
+    srpms: []
+  tasks:
+    - include_role:
+        name: ensure-mock
+
+    - include_role:
+        name: filter-artifacts
+      when: zuul.artifacts is defined
+
+    - include_role:
+        name: source-to-tarball
+      vars:
+        zuul:
+          project: "{{ item }}"
+      loop: "{{ project_deps_to_build | union(project_to_build) }}"
+      when: build_from_source
+
+    - include_role:
+        name: mock-srpm-build
+      vars:
+        zuul:
+          project: "{{ item }}"
+      loop: "{{ project_deps_to_build | union(project_to_build) }}"
+
+    - include_role:
+        name: mock-build
+      vars:
+        srpm: "{{ item }}"
+      loop: "{{ srpms }}"