From a089716f92cf2b09aff3dba2b6c1e71b890e8a75 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Aug 01 2016 11:09:50 +0000 Subject: [PATCH 1/6] update pungi-trigger to work with current fm-orchestrator * listen to build.state_name "ready" rather than commit to repo * get rid of publish which will be done separately --- diff --git a/pungi-trigger.py b/pungi-trigger.py index 0a1f96d..122991f 100755 --- a/pungi-trigger.py +++ b/pungi-trigger.py @@ -8,23 +8,18 @@ from __future__ import print_function import os import sys import subprocess as sp -import tempfile -import shutil - -import yaml import fedmsg import fedmsg.config import fedmsg.meta +import pdc_client config = fedmsg.config.load_config() fedmsg.meta.make_processors(**config) - -distgit_url = 'http://pkgs.stg.fedoraproject.org' -target = 'org.fedoraproject.stg.git.receive' -pubdir = os.environ['PUBDIR'] -puburl = os.environ['PUBURL'] +target_topic = 'org.fedoraproject.dev.rida.module.state.change' +pdcurl = os.environ['PDCURL'] +repodir = os.environ['REPODIR'] def log(*args): @@ -55,15 +50,6 @@ def run(cmd): return lines - -def get_static_manifest_location(lines): - telltale = "Wrote rpm static manifest into" - for line in lines: - if telltale in line: - return line.split(telltale)[-1].strip() - raise ValueError("Could not find %r in output." % telltale) - - def get_built_repo_location(lines): telltale = "as destination dir" for line in lines: @@ -71,122 +57,45 @@ def get_built_repo_location(lines): return line.split(telltale)[0].strip().split()[-1] raise ValueError("Could not find %r in output." % telltale) -def publish_directory(path, link=None): - log(" Publishing directory:", path) - tail = path.split('/')[-1] - dest = os.path.join(pubdir, tail) - - log(" Moving content to", dest) - if os.path.exists(dest): - log(" ...first removing old content at", dest) - shutil.rmtree(dest) - shutil.move(path, dest) - - if link: - link = os.path.join(pubdir, link) - log(" Managing link at ", link) - try: - os.remove(link) - except OSError: - pass - os.symlink(dest, link) - - -def is_modularity_event(msg, config): - # XXX - HACK - ideally, we would check a 'namespace' field in the git - # message here, but we don't have such a field yet. - # See -- https://fedorahosted.org/fedora-infrastructure/ticket/5245 - - # Instead, for now, just operate on a hardcoded set. - known_modules = set([ - "testmodule", - "core", - ]) - - modules = fedmsg.meta.msg2packages(msg, **config) - return bool(set.intersection(modules, known_modules)) - +def get_tag_from_pdc(pdc_proxy, name, version, release): + tv = pdc_proxy.unreleasedvariants(page_size=-1, variant_id=name, variant_version=version, variant_release=release) + if not tv: + return None + return tv[0]['koji_tag'] def main(): - log("* Listening for messages on topic %r" % target) - for _, _, topic, msg in fedmsg.tail_messages(topic=target, **config): + log("* Connecting to PDC %s" % pdcurl) + pdc_proxy = pdc_client.PDCClient(pdcurl, insecure=True, develop=True) + + log("* Listening for messages on topic %r" % target_topic) + for _, _, topic, msg in fedmsg.tail_messages(topic=target_topic, **config): + print(msg) idx = msg['msg_id'] log("Saw a message", topic, idx) - - if not is_modularity_event(msg, config): - # Throw it away - log("Dropping", topic, idx) + if msg['msg']['state_name'] != "ready": continue + # Run signed repo log("Working with", topic, idx) - - # There should only be one package (aka, 'module') here.. - module = list(fedmsg.meta.msg2packages(msg, **config))[0] - - # Create a temp space to start the work.. - checkout = tempfile.mktemp() - try: - # Get the metadata that changed - link = '%s/cgit/modules/%s.git' % (distgit_url, module) - lines = run(['git', 'clone', link, checkout]) - - files = os.listdir(checkout) - files = [f for f in files if f.endswith('.yaml')] - - for filename in files: - # Run depsolver - lines = run([ - 'pungi-gather', - '--arch', 'x86_64', - '--config', checkout + '/' + filename, - '--target-dir', os.environ['TARGET_DIR'], - '--source-repo-from-path', os.environ['SOURCE_REPO'], - ]) - - # Run linker/createrepo with static manifest - manifest = get_static_manifest_location(lines) - log(" Got this manifest for pungi-createrepo", manifest) - lines = run([ - 'pungi-createrepo', - '--target-dir', os.environ['TARGET_DIR'], - '--source-repo-from-path', os.environ['SOURCE_REPO'], - '--static-content-manifest', manifest, - '--arch', 'x86_64', - '--extra-file', 'modulemd.yaml,' + manifest + '/fm-metadata.yaml' - ]) - - # Lastly, move the built module into a web accessible location. - repopath = get_built_repo_location(lines) - log(" Got this repopath for web publication", repopath) - - publish_directory(manifest) - publish_directory(repopath, link=module) - - # And, final step, update the modules.yaml index with a note. - # Read it - indexfile = os.path.join(pubdir, 'modules.yaml') - log(" Modifying indexfile at", indexfile) - try: - with open(indexfile, 'rb') as f: - index = yaml.load(f.read().decode('utf-8')) - index = index or {} # Protect against 'None' - except IOError: - index = {} - - # Modify it - suffix = 'rpms/repodata/modulemd.yaml' - index[module.decode('utf-8')] = \ - os.path.join(puburl, module, suffix).decode('utf-8') - - # And write it back. - with open(indexfile, 'wb') as f: - f.write(yaml.dump(index).encode('utf-8')) - finally: - try: - shutil.rmtree(checkout) - except: - pass - + sigkeys = ['unsigned', ] # XXX: fix this + name = msg['msg']['name'] + version = msg['msg']['version'] + release = msg['msg']['release'] + tag = get_tag_from_pdc(pdc_proxy, name, version, release) + if not tag: + log("ERROR: Could get tag from PDC for module %s-%s-%s. Skipping repo creation." % (name, version, release)) + continue + # Should be exception safe + cmd = [ + 'pungi-signed-repo-prototype', + '--target-dir', os.environ['REPODIR'], + '--koji-profile', os.environ['KOJI_PROFILE'], + tag, + ] + cmd.extend(sigkeys) # multiple + print(cmd) + log("Attemping to execute %s" % " ".join(cmd)) + lines = run(cmd) if __name__ == '__main__': sanity_check() From d1bce7210c7d45f951e92ea50d5832410eb9ab73 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Aug 01 2016 11:11:42 +0000 Subject: [PATCH 2/6] Update default systemd.config to match our staging area --- diff --git a/systemd/pungi-modularity.env b/systemd/pungi-modularity.env index 8adfc09..482b17d 100644 --- a/systemd/pungi-modularity.env +++ b/systemd/pungi-modularity.env @@ -1,8 +1,11 @@ # Just an example env file. In prod/stg, we'll use some real values here. # See https://pagure.io/pungi-modularity/ -export MODULARITY_DIR=/home/threebean/devel/pungi-modularity/modularity -export PYTHONPATH=$MODULARITY_DIR/productmd:$MODULARITY_DIR/pungi:$MODULARITY_DIR/pungi-gather:$MODULARITY_DIR/fm-metadata -export PATH=$MODULARITY_DIR/pungi-gather/bin:$MODULARITY_DIR/pungi/bin:$PATH -export TARGET_DIR=~/RHWorkdir -export SOURCE_REPO=fedora-24-alpha-x86_64 +MODULARITY_DIR="/srv/modularity/pungi-trigger-repos" +# Libhif is required by pungi-signed-repo-prototype +export PYTHONPATH="$MODULARITY_DIR/pungi-prototype:$MODULARITY_DIR/libhif/build/python/hawkey:$MODULARITY_DIR/dnf:$MODULARITY_DIR/modulemd" +export LD_LIBRARY_PATH="$MODULARITY_DIR/libhif/build/libhif" +export PATH=$MODULARITY_DIR/pungi-prototype/bin:$PATH +export REPODIR=/srv/modularity/repos +export PDCURL=http://modularity.fedorainfracloud.org:8080/rest_api/v1/ +export KOJI_PROFILE=koji From 741073b7a7d66267b31415e2cc776a2865348288 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Aug 01 2016 11:16:44 +0000 Subject: [PATCH 3/6] Merge with lkocman/code on stage --- diff --git a/pungi-trigger.py b/pungi-trigger.py index 0a1f96d..122991f 100755 --- a/pungi-trigger.py +++ b/pungi-trigger.py @@ -8,23 +8,18 @@ from __future__ import print_function import os import sys import subprocess as sp -import tempfile -import shutil - -import yaml import fedmsg import fedmsg.config import fedmsg.meta +import pdc_client config = fedmsg.config.load_config() fedmsg.meta.make_processors(**config) - -distgit_url = 'http://pkgs.stg.fedoraproject.org' -target = 'org.fedoraproject.stg.git.receive' -pubdir = os.environ['PUBDIR'] -puburl = os.environ['PUBURL'] +target_topic = 'org.fedoraproject.dev.rida.module.state.change' +pdcurl = os.environ['PDCURL'] +repodir = os.environ['REPODIR'] def log(*args): @@ -55,15 +50,6 @@ def run(cmd): return lines - -def get_static_manifest_location(lines): - telltale = "Wrote rpm static manifest into" - for line in lines: - if telltale in line: - return line.split(telltale)[-1].strip() - raise ValueError("Could not find %r in output." % telltale) - - def get_built_repo_location(lines): telltale = "as destination dir" for line in lines: @@ -71,122 +57,45 @@ def get_built_repo_location(lines): return line.split(telltale)[0].strip().split()[-1] raise ValueError("Could not find %r in output." % telltale) -def publish_directory(path, link=None): - log(" Publishing directory:", path) - tail = path.split('/')[-1] - dest = os.path.join(pubdir, tail) - - log(" Moving content to", dest) - if os.path.exists(dest): - log(" ...first removing old content at", dest) - shutil.rmtree(dest) - shutil.move(path, dest) - - if link: - link = os.path.join(pubdir, link) - log(" Managing link at ", link) - try: - os.remove(link) - except OSError: - pass - os.symlink(dest, link) - - -def is_modularity_event(msg, config): - # XXX - HACK - ideally, we would check a 'namespace' field in the git - # message here, but we don't have such a field yet. - # See -- https://fedorahosted.org/fedora-infrastructure/ticket/5245 - - # Instead, for now, just operate on a hardcoded set. - known_modules = set([ - "testmodule", - "core", - ]) - - modules = fedmsg.meta.msg2packages(msg, **config) - return bool(set.intersection(modules, known_modules)) - +def get_tag_from_pdc(pdc_proxy, name, version, release): + tv = pdc_proxy.unreleasedvariants(page_size=-1, variant_id=name, variant_version=version, variant_release=release) + if not tv: + return None + return tv[0]['koji_tag'] def main(): - log("* Listening for messages on topic %r" % target) - for _, _, topic, msg in fedmsg.tail_messages(topic=target, **config): + log("* Connecting to PDC %s" % pdcurl) + pdc_proxy = pdc_client.PDCClient(pdcurl, insecure=True, develop=True) + + log("* Listening for messages on topic %r" % target_topic) + for _, _, topic, msg in fedmsg.tail_messages(topic=target_topic, **config): + print(msg) idx = msg['msg_id'] log("Saw a message", topic, idx) - - if not is_modularity_event(msg, config): - # Throw it away - log("Dropping", topic, idx) + if msg['msg']['state_name'] != "ready": continue + # Run signed repo log("Working with", topic, idx) - - # There should only be one package (aka, 'module') here.. - module = list(fedmsg.meta.msg2packages(msg, **config))[0] - - # Create a temp space to start the work.. - checkout = tempfile.mktemp() - try: - # Get the metadata that changed - link = '%s/cgit/modules/%s.git' % (distgit_url, module) - lines = run(['git', 'clone', link, checkout]) - - files = os.listdir(checkout) - files = [f for f in files if f.endswith('.yaml')] - - for filename in files: - # Run depsolver - lines = run([ - 'pungi-gather', - '--arch', 'x86_64', - '--config', checkout + '/' + filename, - '--target-dir', os.environ['TARGET_DIR'], - '--source-repo-from-path', os.environ['SOURCE_REPO'], - ]) - - # Run linker/createrepo with static manifest - manifest = get_static_manifest_location(lines) - log(" Got this manifest for pungi-createrepo", manifest) - lines = run([ - 'pungi-createrepo', - '--target-dir', os.environ['TARGET_DIR'], - '--source-repo-from-path', os.environ['SOURCE_REPO'], - '--static-content-manifest', manifest, - '--arch', 'x86_64', - '--extra-file', 'modulemd.yaml,' + manifest + '/fm-metadata.yaml' - ]) - - # Lastly, move the built module into a web accessible location. - repopath = get_built_repo_location(lines) - log(" Got this repopath for web publication", repopath) - - publish_directory(manifest) - publish_directory(repopath, link=module) - - # And, final step, update the modules.yaml index with a note. - # Read it - indexfile = os.path.join(pubdir, 'modules.yaml') - log(" Modifying indexfile at", indexfile) - try: - with open(indexfile, 'rb') as f: - index = yaml.load(f.read().decode('utf-8')) - index = index or {} # Protect against 'None' - except IOError: - index = {} - - # Modify it - suffix = 'rpms/repodata/modulemd.yaml' - index[module.decode('utf-8')] = \ - os.path.join(puburl, module, suffix).decode('utf-8') - - # And write it back. - with open(indexfile, 'wb') as f: - f.write(yaml.dump(index).encode('utf-8')) - finally: - try: - shutil.rmtree(checkout) - except: - pass - + sigkeys = ['unsigned', ] # XXX: fix this + name = msg['msg']['name'] + version = msg['msg']['version'] + release = msg['msg']['release'] + tag = get_tag_from_pdc(pdc_proxy, name, version, release) + if not tag: + log("ERROR: Could get tag from PDC for module %s-%s-%s. Skipping repo creation." % (name, version, release)) + continue + # Should be exception safe + cmd = [ + 'pungi-signed-repo-prototype', + '--target-dir', os.environ['REPODIR'], + '--koji-profile', os.environ['KOJI_PROFILE'], + tag, + ] + cmd.extend(sigkeys) # multiple + print(cmd) + log("Attemping to execute %s" % " ".join(cmd)) + lines = run(cmd) if __name__ == '__main__': sanity_check() diff --git a/systemd/pungi-modularity.env b/systemd/pungi-modularity.env index 8adfc09..482b17d 100644 --- a/systemd/pungi-modularity.env +++ b/systemd/pungi-modularity.env @@ -1,8 +1,11 @@ # Just an example env file. In prod/stg, we'll use some real values here. # See https://pagure.io/pungi-modularity/ -export MODULARITY_DIR=/home/threebean/devel/pungi-modularity/modularity -export PYTHONPATH=$MODULARITY_DIR/productmd:$MODULARITY_DIR/pungi:$MODULARITY_DIR/pungi-gather:$MODULARITY_DIR/fm-metadata -export PATH=$MODULARITY_DIR/pungi-gather/bin:$MODULARITY_DIR/pungi/bin:$PATH -export TARGET_DIR=~/RHWorkdir -export SOURCE_REPO=fedora-24-alpha-x86_64 +MODULARITY_DIR="/srv/modularity/pungi-trigger-repos" +# Libhif is required by pungi-signed-repo-prototype +export PYTHONPATH="$MODULARITY_DIR/pungi-prototype:$MODULARITY_DIR/libhif/build/python/hawkey:$MODULARITY_DIR/dnf:$MODULARITY_DIR/modulemd" +export LD_LIBRARY_PATH="$MODULARITY_DIR/libhif/build/libhif" +export PATH=$MODULARITY_DIR/pungi-prototype/bin:$PATH +export REPODIR=/srv/modularity/repos +export PDCURL=http://modularity.fedorainfracloud.org:8080/rest_api/v1/ +export KOJI_PROFILE=koji From ee753775bb9244534a97d267242a2d861f28650d Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Aug 02 2016 14:25:23 +0000 Subject: [PATCH 4/6] tweak pungi-modularity.env for current staging infra and pungi-signed-repo --- diff --git a/systemd/pungi-modularity.env b/systemd/pungi-modularity.env index 482b17d..11a6bdc 100644 --- a/systemd/pungi-modularity.env +++ b/systemd/pungi-modularity.env @@ -9,3 +9,5 @@ export PATH=$MODULARITY_DIR/pungi-prototype/bin:$PATH export REPODIR=/srv/modularity/repos export PDCURL=http://modularity.fedorainfracloud.org:8080/rest_api/v1/ export KOJI_PROFILE=koji +export NOTIFICATION_SCRIPT="pungi-signed-repo-fedmsg-notification" +export LINK_TYPE="download" From d4867dc9c7a2102daf5e723053d490074928e26c Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Aug 02 2016 14:26:34 +0000 Subject: [PATCH 5/6] pungi-trigger: supply modulemd from scmurl/fedmsg --- diff --git a/pungi-trigger.py b/pungi-trigger.py index 122991f..74ffaf1 100755 --- a/pungi-trigger.py +++ b/pungi-trigger.py @@ -15,6 +15,11 @@ import fedmsg.config import fedmsg.meta import pdc_client +import tempfile +import glob +import rida.scm # XXX: replace +import shutil + config = fedmsg.config.load_config() fedmsg.meta.make_processors(**config) target_topic = 'org.fedoraproject.dev.rida.module.state.change' @@ -63,39 +68,58 @@ def get_tag_from_pdc(pdc_proxy, name, version, release): return None return tv[0]['koji_tag'] +def checkout_modulemd(scmurl): + scm_obj = rida.scm.SCM(url=scmurl) + + src_dir = scm_obj.checkout(tempfile.mkdtemp(prefix="pungi-trigger")) + mmds = glob.glob("%s/*.yaml" % src_dir) + assert len(mmds) == 1, "Expected exactly 1 modulemd file in git checkout. Got: %s" % mmds + return mmds[0] + def main(): log("* Connecting to PDC %s" % pdcurl) pdc_proxy = pdc_client.PDCClient(pdcurl, insecure=True, develop=True) log("* Listening for messages on topic %r" % target_topic) for _, _, topic, msg in fedmsg.tail_messages(topic=target_topic, **config): - print(msg) - idx = msg['msg_id'] - log("Saw a message", topic, idx) - if msg['msg']['state_name'] != "ready": - continue - - # Run signed repo - log("Working with", topic, idx) - sigkeys = ['unsigned', ] # XXX: fix this - name = msg['msg']['name'] - version = msg['msg']['version'] - release = msg['msg']['release'] - tag = get_tag_from_pdc(pdc_proxy, name, version, release) - if not tag: - log("ERROR: Could get tag from PDC for module %s-%s-%s. Skipping repo creation." % (name, version, release)) - continue - # Should be exception safe - cmd = [ - 'pungi-signed-repo-prototype', - '--target-dir', os.environ['REPODIR'], - '--koji-profile', os.environ['KOJI_PROFILE'], - tag, - ] - cmd.extend(sigkeys) # multiple - print(cmd) - log("Attemping to execute %s" % " ".join(cmd)) - lines = run(cmd) + try: + print(msg) + idx = msg['msg_id'] + log("Saw a message", topic, idx) + if msg['msg']['state_name'] != "ready": + continue + + # Run signed repo + log("Working with", topic, idx) + sigkeys = ['unsigned', ] # XXX: fix this + name = msg['msg']['name'] + version = msg['msg']['version'] + release = msg['msg']['release'] + tag = get_tag_from_pdc(pdc_proxy, name, version, release) + if not tag: + log("ERROR: Could get tag from PDC for module %s-%s-%s. Skipping repo creation." % (name, version, release)) + continue + # Should be exception safe + modulemd_path = checkout_modulemd(msg['msg']['scmurl']) + cmd = [ + 'pungi-signed-repo-prototype', + '--target-dir', os.environ['REPODIR'], + '--koji-profile', os.environ['KOJI_PROFILE'], + '--notification-script', os.environ['NOTIFICATION_SCRIPT'], + '--link-type', os.environ['LINK_TYPE'], + '--modulemd', modulemd_path, + tag, + ] + cmd.extend(sigkeys) # multiple + print(cmd) + log("Attemping to execute %s" % " ".join(cmd)) + lines = run(cmd) + + # Remove checkout of module definition + shutil.rmtree(os.path.dirname(modulemd_path), ignore_errors=True) + except Exception, e: + log("ERROR: %s" % e) + pass if __name__ == '__main__': sanity_check() From 1399f96834fafe186924ba6c4dccc3ed8e41f406 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Aug 02 2016 14:33:12 +0000 Subject: [PATCH 6/6] set endpoint to rida on modularity.fedorainfracloud.org --- diff --git a/fedmsg.d/fed-mod-config.py b/fedmsg.d/fed-mod-config.py index 9c2f4c0..ac9124b 100644 --- a/fedmsg.d/fed-mod-config.py +++ b/fedmsg.d/fed-mod-config.py @@ -3,10 +3,10 @@ config = { 'validate_signatures': False, 'endpoints': { - 'modularity-test-server': [ + 'rida.modularity': [ # This assumes that we fedmsg-relay running with its relay_outbound # endpoint set at port 4001 and that that port is opened in os1. - 'tcp://fed-mod.org:4001', + 'tcp://modularity.fedorainfracloud.org:4001', ], }, }