From 727d6f77c91287b42eda88af0a156d442936e731 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 17 2017 10:44:18 +0000 Subject: [PATCH 1/6] Include the load_from_disk utility script This is the script used to populate pagure's database based on what is on disk and in pkgdb. It is very Fedora-specific and thus belongs more here than in pagure itself. Signed-off-by: Pierre-Yves Chibon --- diff --git a/MANIFEST.in b/MANIFEST.in index c1871c2..d37fdc8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include README.rst include dist_git_auth.py include dist_git_auth_tests.py include pagure_poc.py +include load_from_disk.py recursive-include template * recursive-include scripts * recursive-include static * diff --git a/load_from_disk.py b/load_from_disk.py new file mode 100644 index 0000000..203fac6 --- /dev/null +++ b/load_from_disk.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python + +""" + +Utility script used to load the database based on the data present on disk +for the pagure instance over Fedora's git repositories. + +""" + + +import argparse +import collections +import os + +import requests +import pygit2 +from sqlalchemy.exc import SQLAlchemyError + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + +import pagure +import pagure.lib +import pagure.lib.model + + +PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb/api/' +FOLDER = '/srv/git/repositories/' +MAPPING = { + 'Fedora': 'rpms', + 'Fedora EPEL': 'rpms', + 'Fedora Container': 'container', + 'Fedora EPEL Container': 'container', + 'Fedora Modules': 'modules', +} +BLACKLIST = ['@provenpackager'] + + +def get_poc_of_pkgs(debug=False): + """ Retrieve a dictionary giving the point of contact of each package + in pkgdb. + """ + if debug: + print 'Querying pkgdb' + req = requests.get(PKGDB_URL + 'bugzilla').text + if debug: + print 'Pkgdb data retrieved, getting POC' + pkgs = {} + for line in req.split('\n'): + line = line.strip() + if not line or line.startswith('#'): + continue + line = line.split('|') + if len(line) < 4: + continue + + namespace = MAPPING[line[0].strip()] + if namespace not in pkgs: + pkgs[namespace] = {} + + if line[1] not in pkgs[namespace]: + pkgs[namespace][line[1]] = line[3].replace('group::', '@') + + return pkgs + + +def get_watch_tickets_of_pkgs(debug=False): + """ Retrieve a dictionary giving the watch tickets for each package + in pkgdb. + """ + if debug: + print 'Querying pkgdb' + req = requests.get(PKGDB_URL + 'bugzilla').text + if debug: + print 'Pkgdb data retrieved, getting CC' + pkgs = {} + for line in req.split('\n'): + line = line.strip() + if not line or line.startswith('#'): + continue + line = line.split('|') + if len(line) < 4: + continue + + namespace = MAPPING[line[0].strip()] + if namespace not in pkgs: + pkgs[namespace] = {} + + if line[1] not in pkgs[namespace]: + if line[-1]: + users = line[-1].replace('group::', '@').split(',') + users = [u.strip() for u in users if u.strip()] + pkgs[namespace][line[1]] = users + + return pkgs + + +def get_watch_commits_of_pkgs(debug=False): + """ Retrieve a dictionary giving the watch commit for each package + in pkgdb. + """ + if debug: + print 'Querying pkgdb' + req = requests.get(PKGDB_URL + 'notify?namespace=1').text + if debug: + print 'Pkgdb data retrieved, getting Watch commit' + pkgs = {} + for line in req.split('\n'): + line = line.strip() + if not line or line.startswith('#'): + continue + line = line.split('|') + if len(line) < 3: + continue + + namespace = line[0].strip() + if namespace not in pkgs: + pkgs[namespace] = {} + + if line[1] not in pkgs[namespace]: + if line[-1]: + users = line[-1].replace('group::', '@').split(',') + users = [u.strip() for u in users if u.strip()] + pkgs[namespace][line[1]] = users + + return pkgs + + +def get_maintainers(namespace, debug=False): + """ Retrieve a dictionary giving the maintainers of each package + in pkgdb. + """ + if debug: + print 'Querying pkgdb' + req = requests.get(PKGDB_URL + 'vcs?format=json&namespace=%s' % namespace) + if debug: + print 'Pkgdb data retrieved, getting maintainers' + pkgs = collections.defaultdict(set) + data = req.json()[namespace] + for pkg in data: + pkgers = [] + grps = [] + for br in data[pkg]: + pkgers += [ + p + for p in data[pkg][br]['commit']['people']] + grps += [ + '@%s' % g + for g in data[pkg][br]['commit']['groups'] + if g != 'provenpackager' + ] + pkgs[pkg].update(pkgers) + pkgs[pkg].update(grps) + + return pkgs + + +def set_up_users(pocs, maintainers, debug=False): + """ Add the users and groups in the DB. + """ + if debug: + print 'Adding the user to the DB' + + all_users = set() + + for ns in maintainers: + for user_set in maintainers[ns].values(): + all_users.update(user_set) + + for ns in pocs: + all_users.update(set(pocs[ns].values())) + + if not 'releng' in all_users: + pagure.lib.set_up_user( + session=pagure.SESSION, + username='releng', + fullname='Release Engineering', + default_email='releng@fedoraproject.org', + ) + pagure.SESSION.commit() + + for user in sorted(all_users): + if debug: + print user + try: + if user.startswith('@') or user.startswith('group::'): + if user.startswith('@'): + name = user[1:] + else: + name = user[7:] + pagure.lib.add_group( + session=pagure.SESSION, + group_name=name, + display_name=name, + description='Group %s' % name, + group_type='user', + user='releng', + is_admin=False, + blacklist=pagure.APP.config['BLACKLISTED_GROUPS'], + ) + pagure.SESSION.commit() + else: + pagure.lib.set_up_user( + session=pagure.SESSION, + username=user, + fullname=user, + default_email='%s@fedoraproject.org' % user, + ) + pagure.SESSION.commit() + except SQLAlchemyError as err: + pagure.SESSION.rollback() + print 'ERROR with user %s' % user + print err + + + +def main(pocs, maintainers, ccs, watch_commits, folder, namespace, debug=False): + """ + Logic: + - Query the list of maintainer/PoC from pkgdb + - Set the users in the database based on the PoC retrieved + - Browse the directory + - For each git in the directory, create the project with the correct POC + + """ + + for project in sorted(os.listdir(folder)): + if not project.endswith('.git'): + if debug: + print ' -skip: not a git repository' + continue + + name = project.rsplit('.git', 1)[0] + fullname = '%s/%s' % (namespace, name) + + if name not in pocs: + if debug: + print ' -no pocs: setting poc to `orphan`' + pocs[name] = 'orphan' + + if debug: + print project, pocs[name] + + try: + if fullname in pagure.APP.config['BLACKLISTED_PROJECTS']: + raise pagure.exceptions.RepoExistsException( + 'No project "%s" are allowed to be created due to ' + 'potential conflicts in URLs with pagure itself' % name + ) + + pkg_poc = pocs[name] + if pkg_poc.startswith('@'): + for user in maintainers[name]: + user = user.strip() + if not user: + continue + if not user.startswith('@'): + if debug: + print ' %s name had for POC: %s, switching to %s' % ( + name, pkg_poc, user) + pkg_poc = user + break + + user_obj = pagure.lib.get_user(pagure.SESSION, pkg_poc) + allowed_prefix = pagure.APP.config[ + 'ALLOWED_PREFIX'] + [grp for grp in user_obj.groups] + + first_part, _, second_part = name.partition('/') + if second_part and first_part not in allowed_prefix: + raise pagure.exceptions.PagureException( + 'The prefix of your project must be in the list of ' + 'allowed prefixes set by the admins of this pagure ' + 'instance, or the name of a group of which you are a ' + 'member.' + ) + + gitfolder = pagure.APP.config['GIT_FOLDER'] + requestfolder = pagure.APP.config['REQUESTS_FOLDER'] + requestfolder = pagure.APP.config['REQUESTS_FOLDER'] + + project = pagure.lib.model.Project( + name=name, + namespace=namespace, + description='The %s %s' % (name, namespace), + url=None, + avatar_email=None, + user_id=user_obj.id, + parent_id=None, + hook_token=pagure.lib.login.id_generator(40) + ) + pagure.SESSION.add(project) + # Make sure we won't have SQLAlchemy error before we create + # the repo + pagure.SESSION.flush() + + if name in maintainers: + for maintainer in maintainers[name]: + maintainer = maintainer.strip() + if not maintainer: + continue + if debug: + print ' - %s' % maintainer + if maintainer in project.contributors['commit']: + if debug: + print ' - %s has already commit' % maintainer + continue + try: + if maintainer.startswith('@') \ + or maintainer.startswith('group::'): + if maintainer.startswith('@'): + name = maintainer[1:] + else: + name = maintainer[7:] + out = pagure.lib.add_group_to_project( + session=pagure.SESSION, + project=project, + new_group=name, + user='releng', + access='commit', + is_admin=True) + if debug: + print ' -', out + else: + out = pagure.lib.add_user_to_project( + session=pagure.SESSION, + project=project, + new_user=maintainer, + user='releng', + access='commit') + if debug: + print ' -', out + pagure.SESSION.commit() + except pagure.exceptions.PagureException as err: + print 'ERROR with project %s' % fullname + print err + + # Set-up the watch-ticket flags + if name in ccs: + for maintainer in ccs[name]: + maintainer = maintainer.strip() + if not maintainer: + continue + if debug: + print ' - %s' % maintainer + + if maintainer.startswith('@'): + if debug: + print ' Skipping group' + continue + + watch = '1' + if maintainer in maintainers[name]: + watch = '3' + + try: + out = pagure.lib.update_watch_status( + session=pagure.SESSION, + project=project, + user=maintainer, + watch=watch + ) + if debug: + print ' -', out + except pagure.exceptions.PagureException as err: + print 'ERROR with project %s' % fullname + print err + + # Set-up the watch-commits flags + if name in watch_commits: + for maintainer in watch_commits[name]: + maintainer = maintainer.strip() + if not maintainer: + continue + if debug: + print ' - %s' % maintainer + + if maintainer.startswith('@'): + if debug: + print ' Skipping group' + continue + + watch = '2' + if maintainer in maintainers.get(name, []) \ + or maintainer in ccs.get(name, []): + watch = '3' + + try: + out = pagure.lib.update_watch_status( + session=pagure.SESSION, + project=project, + user=maintainer, + watch=watch + ) + if debug: + print ' -', out + except pagure.exceptions.PagureException as err: + print 'ERROR with project %s' % fullname + print err + + gitrepo = os.path.join(gitfolder, '%s.git' % fullname) + http_clone_file = os.path.join(gitrepo, 'git-daemon-export-ok') + if not os.path.exists(http_clone_file): + with open(http_clone_file, 'w') as stream: + pass + + requestrepo = os.path.join(requestfolder, project.path) + if not os.path.exists(requestrepo): + pygit2.init_repository( + requestrepo, bare=True, + mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) + elif debug: + print ' - The requests repo "%s" already exists' % ( + project.path) + pass + + pagure.SESSION.commit() + except pagure.exceptions.PagureException as err: + print 'ERROR with project %s' % fullname + print err + except SQLAlchemyError as err: # pragma: no cover + pagure.SESSION.rollback() + print 'ERROR (DB) with project %s' % project + print err + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Script creating projects on pagure based on the git ' + 'repos present in the specified folder and the pkgdb information.' + ) + parser.add_argument( + '--debug', dest='debug', action='store_true', default=False, + help='Print the debugging output') + + args = parser.parse_args() + + pocs = get_poc_of_pkgs(debug=args.debug) + ccs = get_watch_tickets_of_pkgs(debug=args.debug) + watch_commit = get_watch_commits_of_pkgs(debug=args.debug) + maintainers = {} + for namespace in ['rpms', 'modules', 'container']: + maintainers[namespace] = get_maintainers(namespace, debug=args.debug) + set_up_users(pocs, maintainers, debug=args.debug) + + for namespace in ['rpms', 'modules', 'container']: + + folder = os.path.join(FOLDER, namespace) + main( + pocs=pocs[namespace], + maintainers=maintainers[namespace], + ccs=ccs[namespace], + watch_commits=watch_commit[namespace], + folder=folder, + namespace=namespace, + debug=args.debug + ) From b358b90dc51f176c7d154b6539887eed0c78281f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 17 2017 10:44:18 +0000 Subject: [PATCH 2/6] Include the sync_fas_group utility script This script is used to sync group membership from FAS to pagure for all the groups set/used in pagure. Signed-off-by: Pierre-Yves Chibon --- diff --git a/MANIFEST.in b/MANIFEST.in index d37fdc8..426e373 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,7 @@ include dist_git_auth.py include dist_git_auth_tests.py include pagure_poc.py include load_from_disk.py +include sync_fas_group.py recursive-include template * recursive-include scripts * recursive-include static * diff --git a/sync_fas_group.py b/sync_fas_group.py new file mode 100644 index 0000000..2273d0f --- /dev/null +++ b/sync_fas_group.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python + +""" + +Utility script used to load the database based on the data present on disk +for the pagure instance over Fedora's git repositories. + +""" + + +import argparse +import collections +import getpass +import logging +import os + +import requests +from fedora.client import AccountSystem + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + +import pagure +import pagure.lib +import pagure.lib.model + +FAS = 'https://admin.fedoraproject.org/accounts' +URL = 'https://src.fedoraproject.org/' + +_log = logging.getLogger(__name__) + +_fas_username = raw_input("FAS user: ") +_fas_password = getpass.getpass("FAS password: ") +FASCLIENT = AccountSystem( + FAS, + username=_fas_username, + password=_fas_password, +) + +groups = [ + "alt-gtk-de-sig", + "astro-sig", + "eclipse-sig", + "gnome-sig", + "infra-sig", + "kde-sig", + "lxqt-sig", + "neuro-sig", + "nodejs-sig", + "python-sig", + "qa-tools-sig", + "robotics-sig", + "rpm-software-management-sig", + "ruby-packagers-sig", + "rust-sig", + "virtmaint-sig", + "packager", +] + + +def get_pagure_group(): + """ Get the membership info for all the groups in pagure. + Returns dict of {group_name: [members]} + """ + output = collections.defaultdict(set) + base_url = URL.rstrip('/') + req = requests.get(url='%s/api/0/groups' % base_url, timeout=30) + data = req.json() + for grp in data['groups']: + req = requests.get( + url='%s/api/0/group/%s' % (base_url, grp), timeout=30) + data = req.json() + output[grp].update(data['members']) + output[grp].update([data['creator']['name']]) + + return output + + +def get_fas_groups(groups): + """ Return a dict containing for each group specified in the list passed + as argument the list of its members. + """ + output = {} + for group in groups: + members = [it.username for it in FASCLIENT.group_members(group)] + output[group] = members + return output + + +def main(): + """ Core part of this script. + Logic: + - Get all the groups in pagure/dist-git + - For each group, get the membership + - Query FAS for membership of each group + - Adjust pagure's database + + """ + _log.info('Getting groups from pagure') + pagure_groups = get_pagure_group() + _log.info('Getting membership from FAS') + fas_groups = get_fas_groups(pagure_groups.keys()) + + for group in groups: + _log.info('Processing group %s' % (group)) + upstream = set(fas_groups.get(group, [])) + downstream = set(pagure_groups.get(group, [])) + to_add = upstream - downstream + to_rm = downstream - upstream + if 'releng' in to_rm: + to_rm.remove('releng') + + group_obj = pagure.lib.search_groups( + session=pagure.SESSION, + group_name=group + ) + for user in to_rm: + _log.info(' - Removing user %s from group %s' % (user, group)) + pagure.lib.delete_user_of_group( + session=pagure.SESSION, + username=user, + groupname=group, + user='releng', + is_admin=True, + force=False + ) + pagure.SESSION.commit() + for user in to_add: + _log.info(' + Adding user %s to group %s' % (user, group)) + try: + pagure.lib.add_user_to_group( + session=pagure.SESSION, + username=user, + group=group_obj, + user='releng', + is_admin=True) + except pagure.exceptions.PagureException as err: + _log.error('ERROR: %s', err) + pagure.SESSION.commit() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Script syncing FAS groups membership to pagure' + ) + parser.add_argument( + '--debug', dest='debug', action='store_true', default=False, + help='Print the debugging output') + + args = parser.parse_args() + logging.basicConfig() + if args.debug: + _log.setLevel(logging.DEBUG) + else: + _log.setLevel(logging.WARNING) + + main() From 6a489d45b11d69bc0a6054dc27a303167f212eb0 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 17 2017 10:44:18 +0000 Subject: [PATCH 3/6] Add utility script to migrate a single project from pkgdb to pagure This is/has been useful to polish the migration as some project originally failed their migration, this allowed a more hand-held migration for them. Signed-off-by: Pierre-Yves Chibon --- diff --git a/load_single.py b/load_single.py new file mode 100644 index 0000000..4026061 --- /dev/null +++ b/load_single.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python + +""" + +Utility script used to sync a single project from pkgdb to pagure with the +added information provided. + +This script relies on what having the main admin or group already set on +pagure. + +""" + + +import argparse +import collections +import logging +import os + +import requests +import pygit2 +from sqlalchemy.exc import SQLAlchemyError + +_log = logging.getLogger(__name__) + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + +import pagure +import pagure.lib +import pagure.lib.model + + +PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb/api/' +BLACKLIST = ['@provenpackager', 'group::provenpackager'] + + +def get_pkg_info(namespace, pkg): + """ Retrieve a dictionary giving the maintainers of each package + in pkgdb. + """ + _log.debug('Querying pkgdb') + req = requests.get(PKGDB_URL + 'package/%s/%s' % (namespace, pkg)) + _log.debug('Pkgdb data retrieved, getting ACLs') + acls = collections.defaultdict(set) + data = req.json()['packages'] + for pkg in data: + for acl in pkg['acls']: + if acl['status'] == 'Approved': + acls[acl['acl']].add(acl['fas_name']) + + return acls + + +def main(namespace, project, poc): + """ + Logic: + - Retrieve the pkgdb information about this project + - Create the project in pagure with the provided main admin and the + rest of the information from pkgdb. + + """ + + _log.debug('namespace: %s', namespace) + _log.debug('project: %s', project) + _log.debug('poc: %s', poc) + + pkg_info = get_pkg_info(namespace, project) + + fullname = '%s/%s' % (namespace, project) + try: + if fullname in pagure.APP.config['BLACKLISTED_PROJECTS']: + raise pagure.exceptions.RepoExistsException( + 'No project "%s" are allowed to be created due to ' + 'potential conflicts in URLs with pagure itself' % project + ) + + user_obj = pagure.lib.get_user(pagure.SESSION, poc) + allowed_prefix = pagure.APP.config[ + 'ALLOWED_PREFIX'] + [grp for grp in user_obj.groups] + + if namespace not in allowed_prefix: + raise pagure.exceptions.PagureException( + 'The prefix of your project must be in the list of ' + 'allowed prefixes set by the admins of this pagure ' + 'instance, or the name of a group of which you are a ' + 'member.' + ) + + project = pagure.lib.model.Project( + name=project, + namespace=namespace, + description='The %s %s' % (project, namespace), + url=None, + avatar_email=None, + user_id=user_obj.id, + parent_id=None, + hook_token=pagure.lib.login.id_generator(40) + ) + pagure.SESSION.add(project) + # Make sure we won't have SQLAlchemy error before we create + # the repo + pagure.SESSION.flush() + + _log.info('Set commit and admin access') + for acl in pkg_info: + if acl not in ['commit', 'approveacls']: + continue + + access = 'commit' + if acl == 'approveacls': + access = 'admin' + + for maintainer in pkg_info[acl]: + maintainer = maintainer.strip() + if not maintainer: + continue + _log.info(' Processing: %s', maintainer) + if maintainer in project.contributors['commit']: + _log.debug(' - %s has already commit' % maintainer) + continue + try: + if maintainer.startswith(('@', 'group::')): + if maintainer in BLACKLIST: + continue + if maintainer.startswith('@'): + name = maintainer[1:] + else: + name = maintainer[7:] + + out = pagure.lib.add_group_to_project( + session=pagure.SESSION, + project=project, + new_group=name, + user='releng', + access=access, + is_admin=True) + _log.debug(' - %s', out) + else: + out = pagure.lib.add_user_to_project( + session=pagure.SESSION, + project=project, + new_user=maintainer, + user='releng', + access=access) + _log.debug(' - %s', out) + pagure.SESSION.commit() + except pagure.exceptions.PagureException as err: + print 'ERROR with project %s' % fullname + print err + + _log.info('Set watch ticket flags') + # Set-up the watch-ticket flags + for maintainer in pkg_info['watchbugzilla']: + maintainer = maintainer.strip() + if not maintainer: + continue + _log.info(' - %s' % maintainer) + + if maintainer.startswith(('@', 'group::')): + _log.debug(' Skipping group') + continue + + watch = '1' + if maintainer in pkg_info['watchcommits']: + watch = '3' + + try: + out = pagure.lib.update_watch_status( + session=pagure.SESSION, + project=project, + user=maintainer, + watch=watch + ) + _log.info(' - %s', out) + except pagure.exceptions.PagureException as err: + print 'ERROR with project %s' % fullname + print err + + _log.info('Set watch commit flags') + # Set-up the watch-commits flags + for maintainer in pkg_info['watchcommits']: + maintainer = maintainer.strip() + if not maintainer: + continue + _log.debug(' - %s' % maintainer) + + if maintainer.startswith(('@', 'group::')): + _log.debug(' Skipping group') + continue + + watch = '2' + if maintainer in pkg_info['watchbugzilla']: + watch = '3' + + try: + out = pagure.lib.update_watch_status( + session=pagure.SESSION, + project=project, + user=maintainer, + watch=watch + ) + _log.info(' - %s', out) + except pagure.exceptions.PagureException as err: + print 'ERROR with project %s' % fullname + print err + + gitfolder = pagure.APP.config['GIT_FOLDER'] + requestfolder = pagure.APP.config['REQUESTS_FOLDER'] + + gitrepo = os.path.join(gitfolder, '%s.git' % fullname) + http_clone_file = os.path.join(gitrepo, 'git-daemon-export-ok') + if not os.path.exists(http_clone_file): + with open(http_clone_file, 'w'): + pass + + requestrepo = os.path.join(requestfolder, project.path) + if not os.path.exists(requestrepo): + pygit2.init_repository( + requestrepo, bare=True, + mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) + else: + _log.info(' - The requests repo "%s" already exists' % ( + project.path)) + pass + + pagure.SESSION.commit() + except pagure.exceptions.PagureException as err: + print 'ERROR with project %s' % fullname + print err + except SQLAlchemyError as err: # pragma: no cover + pagure.SESSION.rollback() + print 'ERROR (DB) with project %s' % project + print err + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Script creating a single projects on pagure based on ' + 'provided information and what is in pkgdb.' + ) + parser.add_argument( + 'namespace', help='The namespace of the project') + parser.add_argument( + 'project', help='The name of the project') + parser.add_argument( + '--poc', dest='poc', help='The main admin of the project') + parser.add_argument( + '--debug', dest='debug', action='store_true', default=False, + help='Print the debugging output') + + args = parser.parse_args() + + main( + namespace=args.namespace, + project=args.project, + poc=args.poc + ) From 46325752c06588f3c5fe0b25a3ff319cfd04c6a2 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 17 2017 10:44:18 +0000 Subject: [PATCH 4/6] Add script to sync all the ACLs from pkgdb to pagure This is the script that takes over the work done by the load_from_disk script which was not doing a really good job ACLs wise. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pkgdb2pagure_acls.py b/pkgdb2pagure_acls.py new file mode 100644 index 0000000..d3b120a --- /dev/null +++ b/pkgdb2pagure_acls.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python + +""" +This script loads in pagure the ACLs exported from pkgdb to ensure help the +transition. + + +Export the pkgdb ACL database using: +```` +COPY ( + SELECT DISTINCT package.namespace, + package.name, + package_listing_acl.fas_name, + package_listing_acl.acl, + package_listing_acl.status + FROM package, package_listing, package_listing_acl + WHERE package_listing_acl.packagelisting_id = package_listing.id + AND package_listing.package_id = package.id + ORDER BY package.namespace, package.name +) TO '/tmp/pkgdb_acls.csv' delimiter ',' CSV header; +```` + +""" +from __future__ import print_function + +import time +import os + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print('Using configuration file `/etc/pagure/pagure.cfg`') + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + +import pagure +import pagure.lib +import pagure.lib.model + + +ACL_FILE = '/tmp/pkgdb_acls.csv' + + +def main(): + with open(ACL_FILE) as stream: + data = [row.strip().split(',') for row in stream] + + for row in data: + print(row) + namespace, package, packager, acl, status = row + + # Drop all the commit access, these should be already good + if acl == 'commit': + continue + + # Check if the package exists + pkg = pagure.get_authorized_project( + pagure.SESSION, namespace=namespace, project_name=package) + if not pkg: + print('ERROR: project: %s/%s not found' % (namespace, package)) + continue + + # Check if the packager exists + user = pagure.lib.search_user(pagure.SESSION, username=packager) + if not user: + print('ERROR: packager: %s not found' % packager) + continue + + if acl == 'approveacls' and status == 'Approved': + try: + print(' Making %s admin on %s' % (packager, pkg.fullname)) + pagure.lib.add_user_to_project( + session=pagure.SESSION, + project=pkg, + new_user=packager, + user='releng', + access='admin', + required_groups=None) + except pagure.exceptions.PagureException as err: + print('ERROR: error while making the user %s an admin' % packager) + print(err) + continue + elif acl.startswith('watch'): + watch_levels = pagure.lib.get_watch_level_on_repo( + pagure.SESSION, + packager, + package, + namespace=namespace) + print(' ', acl, status, watch_levels) + + """watch status can be: + -1: reset the watch status to default + 0: unwatch, don't notify the user of anything + 1: watch issues and PRs + 2: watch commits + 3: watch issues, PRs and commits + """ + watch = None + if acl == 'watchbugzilla': + if 'issues' in watch_levels \ + and status not in ['Approved', 'Awaiting Review']: + # Remove watch bugzilla + print(' Removing watch bugzilla/issues') + watch = '0' + if 'commits' in watch_levels: + watch = '2' + elif 'issues' not in watch_levels \ + and status == 'Approved': + # Add watch bugzilla + print(' Adding watch bugzilla') + watch = '1' + if 'commits' in watch_levels: + watch = '3' + else: + print(' Nothing to do for watch bugzilla') + + elif acl == 'watchcommits': + if 'commits' in watch_levels \ + and status not in ['Approved', 'Awaiting Review']: + # Remove watch commits + print(' Removing watch commits') + watch = '0' + if 'issues' in watch_levels: + watch = '1' + elif 'commits' not in watch_levels \ + and status == 'Approved': + # Add watch commits + print(' Adding watch commits') + watch = '2' + if 'issues' in watch_levels: + watch = '3' + else: + print(' Nothing to do for watch commits') + + if watch: + print( + 'Setting watch level: %s to %s for %s' % ( + watch, packager, pkg.fullname)) + pagure.lib.update_watch_status( + session=pagure.SESSION, + project=pkg, + user=packager, + watch=watch) + + pagure.SESSION.commit() + + +if __name__ == '__main__': + import sys + sys.exit(main()) From 5b35c100dffbdf4c8ed726fd5102c7115a53c2af Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 17 2017 10:44:18 +0000 Subject: [PATCH 5/6] Include pkgdb2pagure_acls.py in the releases --- diff --git a/MANIFEST.in b/MANIFEST.in index 426e373..9c42e13 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ include dist_git_auth_tests.py include pagure_poc.py include load_from_disk.py include sync_fas_group.py +include pkgdb2pagure_acls.py recursive-include template * recursive-include scripts * recursive-include static * From 1df7a76a63abc32599c6bc5140830fb884d47f88 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 17 2017 10:44:18 +0000 Subject: [PATCH 6/6] Multiple fixes in the pkgdb2pagure_acls.py script - Fix the SQL query used to extract the ACLs from pkgdb - Try turning off fedmsg notifications as well as turning the project to read-only (doesn't really work for some reason :s) - Do not make admin people that are no longer part of the project Since we migrated all the commits access, it means they lost access in the mean-while - When making people admin of a project, make them ignore the notifications Signed-off-by: Pierre-Yves Chibon --- diff --git a/pkgdb2pagure_acls.py b/pkgdb2pagure_acls.py index d3b120a..53879d5 100644 --- a/pkgdb2pagure_acls.py +++ b/pkgdb2pagure_acls.py @@ -13,10 +13,13 @@ COPY ( package_listing_acl.fas_name, package_listing_acl.acl, package_listing_acl.status - FROM package, package_listing, package_listing_acl + FROM package, package_listing, package_listing_acl, collection WHERE package_listing_acl.packagelisting_id = package_listing.id AND package_listing.package_id = package.id - ORDER BY package.namespace, package.name + AND package_listing.collection_id = collection.id + AND collection.status IN ('Active', 'Under Development') + ORDER BY package.namespace, package.name, + package_listing_acl.fas_name, package_listing_acl.acl ) TO '/tmp/pkgdb_acls.csv' delimiter ',' CSV header; ```` @@ -26,6 +29,8 @@ from __future__ import print_function import time import os +import mock + if 'PAGURE_CONFIG' not in os.environ \ and os.path.exists('/etc/pagure/pagure.cfg'): print('Using configuration file `/etc/pagure/pagure.cfg`') @@ -38,7 +43,10 @@ import pagure.lib.model ACL_FILE = '/tmp/pkgdb_acls.csv' - +mock.patch( + 'pagure.lib.notify.fedmsg_publish', mock.MagicMock(return_value=True)) +mock.patch( + 'pagure.lib.update_read_only_mode', mock.MagicMock(return_value=True)) def main(): with open(ACL_FILE) as stream: data = [row.strip().split(',') for row in stream] @@ -64,7 +72,15 @@ def main(): print('ERROR: packager: %s not found' % packager) continue + if acl == 'approveacls' and status == 'Approved': + # Check if the packager has still commit + committers = [user.username for user in pkg.committers] + if packager not in committers: + print( + ' %s is no longer a committer on %s, let\'s not make ' + 'them admin' % (packager, pkg.fullname)) + continue try: print(' Making %s admin on %s' % (packager, pkg.fullname)) pagure.lib.add_user_to_project( @@ -74,6 +90,11 @@ def main(): user='releng', access='admin', required_groups=None) + pagure.lib.update_watch_status( + session=pagure.SESSION, + project=pkg, + user=packager, + watch='0') except pagure.exceptions.PagureException as err: print('ERROR: error while making the user %s an admin' % packager) print(err)