From 822b31ca455d85b0a7bba68e599975e7cfc81286 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Sep 03 2016 16:29:52 +0000 Subject: [PATCH 1/4] Reduce the number of opened file by cloning and deleting only once the working repository --- diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py index d996050..c1cfb4a 100644 --- a/pagure_importer/utils/git.py +++ b/pagure_importer/utils/git.py @@ -6,7 +6,6 @@ import shutil import os import pygit2 -import tempfile import json import hashlib import werkzeug @@ -16,27 +15,43 @@ from repo import * def get_secure_filename(attachment, filename): filename = '%s-%s' % (hashlib.sha256(str(attachment)).hexdigest(), - werkzeug.secure_filename(str(filename))) + werkzeug.secure_filename(str(filename))) return filename -def update_git(obj, repo_path, repo_folder): - """ Update the given issue in its git. - This method forks the provided repo, add/edit the issue whose file name - is defined by the uid field of the issue and if there are additions/ - changes commit them and push them back to the original repo. - """ +def clone_repo(repo_name, repo_folder): if not repo_folder: return # Get the fork - repopath = os.path.join(repo_folder, repo_path) + repopath = os.path.join(repo_folder, repo_name) # Clone the repo into a temp folder - newpath = tempfile.mkdtemp(prefix='pagure-') + newpath = os.path.join(repo_folder, 'clone-' + repo_name) new_repo = pygit2.clone_repository(repopath, newpath) + return (newpath, new_repo) + + +def push_delete_repo(newpath, new_repo): + + # Push to origin + ori_remote = new_repo.remotes[0] + master_ref = new_repo.lookup_reference('HEAD').resolve() + refname = '%s:%s' % (master_ref.name, master_ref.name) + + PagureRepo.push(ori_remote, refname) + + # Remove the clone + shutil.rmtree(newpath) + +def update_git(obj, newpath, new_repo): + """ Update the given issue in its git. + This method forks the provided repo, add/edit the issue whose file name + is defined by the uid field of the issue and if there are additions/ + changes commit them and push them back to the original repo. + """ file_path = os.path.join(newpath, obj.uid) # Get the current index @@ -110,12 +125,4 @@ def update_git(obj, repo_path, repo_folder): parents) index.write() - # Push to origin - ori_remote = new_repo.remotes[0] - master_ref = new_repo.lookup_reference('HEAD').resolve() - refname = '%s:%s' % (master_ref.name, master_ref.name) - - PagureRepo.push(ori_remote, refname) - - # Remove the clone - shutil.rmtree(newpath) + return new_repo diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index 9c9fd7e..9f1afbd 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -2,7 +2,7 @@ import requests import time import base64 from datetime import datetime -from pagure_importer.utils.git import update_git, get_secure_filename +from pagure_importer.utils.git import * from pagure_importer.utils.models import User, Issue, IssueComment @@ -43,6 +43,7 @@ class TracImporter(): def import_issues(self, repo_name, repo_folder, trac_query='max=0&order=id'): '''Import issues from trac instance using xmlrpc API''' + newpath, new_repo = clone_repo(repo_name, repo_folder) tickets_id = self.request('ticket.query', trac_query) for ticket_id in tickets_id: @@ -60,9 +61,10 @@ class TracImporter(): comments[key].comment += '\n[%s](%s)' % (attach_name, url) pagure_issue.comments.append(comments[key].to_json()) # update the local git repo - update_git(pagure_issue, repo_name, repo_folder) + new_repo = update_git(pagure_issue, newpath, new_repo) print 'Updated ' + repo_name + ' with issue :' + str(ticket_id) +\ '/' + str(tickets_id[-1]) + push_delete_repo(newpath, new_repo) def create_issue(self, ticket_id): From 5c9a7503a11ce4ab2b3ddbd23b04e5b84442ded8 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Sep 04 2016 19:23:19 +0000 Subject: [PATCH 2/4] Modified github importer to use the new git api --- diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index cc61370..c431035 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -37,7 +37,7 @@ class GithubImporter(): except: raise GithubRepoNotFound( 'Repo not found, project name wrong') - + newpath, new_repo = clone_repo(repo_path, repo_folder) for github_issue in repo.get_issues(state=status): # title of the issue @@ -135,4 +135,5 @@ class GithubImporter(): pagure_issue.comments = comments # update the local git repo - update_git(pagure_issue, repo_path, repo_folder) + new_repo = update_git(pagure_issue, newpath, new_repo) + push_delete_repo(newpath, new_repo) From 8dc6ef76d9edb34b54932ff1d096153009de48f3 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Sep 04 2016 19:56:32 +0000 Subject: [PATCH 3/4] Pagure cannot have empty issue reporter, if the trac reporter is not known in FAS, create a dummy user with is trac username --- diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index 9f1afbd..2ff76c2 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -68,8 +68,8 @@ class TracImporter(): def create_issue(self, ticket_id): - trac_ticket_info = self.request('ticket.get', ticket_id) - trac_ticket = trac_ticket_info[3] + trac_ticket_info = self.request('ticket.get', ticket_id) + trac_ticket = trac_ticket_info[3] trac_attachments = self.request('ticket.listAttachments', ticket_id) pagure_attachment = {} @@ -91,9 +91,15 @@ class TracImporter(): if self.fas: pagure_issue_assignee = self.fas.find_fas_user(trac_ticket['owner']) pagure_issue_user = self.fas.find_fas_user(trac_ticket['reporter']) + if not pagure_issue_user.name: + pagure_issue_user = User(name=trac_ticket['reporter'], + fullname=trac_ticket['reporter'], + emails=[trac_ticket['reporter']+'@fedoraproject.org']) else: pagure_issue_assignee = User(name='', fullname='', emails=[]) - pagure_issue_user = User(name='', fullname='', emails=[]) + pagure_issue_user = User(name=trac_ticket['reporter'], + fullname=trac_ticket['reporter'], + emails=[trac_ticket['reporter']+'@fedoraproject.org']) pagure_issue_tags = [] if self.tags: From 94fe805ff2b41c9d5102fb0c1767dfed6f9fbaf2 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Sep 05 2016 18:03:31 +0000 Subject: [PATCH 4/4] fixing pep8 issues --- diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index c431035..a0a37c0 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -2,7 +2,8 @@ from github import Github from pagure_importer.utils import models from pagure_importer.utils import github_get_commentor_email -from pagure_importer.utils.git import update_git +from pagure_importer.utils.git import ( + clone_repo, push_delete_repo, update_git) from pagure_importer.utils.exceptions import ( GithubBadCredentials, GithubRepoNotFound @@ -65,33 +66,29 @@ class GithubImporter(): else: pagure_issue_tags = [] - # few things not supported by github pagure_issue_depends = [] pagure_issue_blocks = [] pagure_issue_is_private = False - # User who created the issue pagure_issue_user = models.User( name=github_issue.user.login, fullname=github_issue.user.name, emails=[github_issue.user.email]) - pagure_issue = models.Issue( id=None, - title = pagure_issue_title, - content = pagure_issue_content, - status = pagure_issue_status, - date_created = pagure_issue_created_at, - user = pagure_issue_user.to_json(), - private = pagure_issue_is_private, - tags = pagure_issue_tags, - depends = pagure_issue_depends, - blocks = pagure_issue_blocks, - assignee = pagure_issue_assignee) - + title=pagure_issue_title, + content=pagure_issue_content, + status=pagure_issue_status, + date_created=pagure_issue_created_at, + user=pagure_issue_user.to_json(), + private=pagure_issue_is_private, + tags=pagure_issue_tags, + depends=pagure_issue_depends, + blocks=pagure_issue_blocks, + assignee=pagure_issue_assignee) # comments on the issue comments = [] @@ -103,7 +100,6 @@ class GithubImporter(): pagure_issue_comment_created_at = comment.created_at pagure_issue_comment_updated_at = comment.updated_at - # No idea what to do with this right now # editor: not supported by github api pagure_issue_comment_parent = None diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index 2ff76c2..78322ff 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -1,21 +1,25 @@ import requests import time import base64 +import sys from datetime import datetime -from pagure_importer.utils.git import * +from pagure_importer.utils.git import ( + clone_repo, get_secure_filename, push_delete_repo, update_git) from pagure_importer.utils.models import User, Issue, IssueComment class TracImporter(): '''Pagure importer for trac instance''' - def __init__(self, project_url, username, password, fasclient=None, tags=False): + def __init__(self, project_url, username, password, + fasclient=None, tags=False): self.url = project_url self.username = username self.password = password self.fas = fasclient self.tags = tags - self.somebody = User(name='somebody', fullname='somebody', emails=['some@body.com']) + self.somebody = User(name='somebody', fullname='somebody', + emails=['some@body.com']) self.reqid = 0 def request(self, method, *args): @@ -23,7 +27,8 @@ class TracImporter(): req = {'params': args, 'method': method, 'id': self.reqid} - resp = requests.post(self.url, json=req, auth=(self.username, self.password)) + resp = requests.post(self.url, json=req, + auth=(self.username, self.password)) resp = resp.json() if resp['id'] != self.reqid: print('ERROR: Invalid response for request! ID does not match') @@ -56,7 +61,8 @@ class TracImporter(): if comments[key].attachment: attach_name = comments[key].attachment project = repo_name.replace('.git', '') - filename = get_secure_filename(pagure_issue.attachment[attach_name], attach_name) + filename = get_secure_filename( + pagure_issue.attachment[attach_name], attach_name) url = '/%s/issue/raw/files/%s' % (project, filename) comments[key].comment += '\n[%s](%s)' % (attach_name, url) pagure_issue.comments.append(comments[key].to_json()) @@ -89,21 +95,25 @@ class TracImporter(): pagure_issue_created_at = self.to_timestamp(trac_ticket_info[1]['__jsonclass__'][1]) if self.fas: - pagure_issue_assignee = self.fas.find_fas_user(trac_ticket['owner']) + pagure_issue_assignee = self.fas.find_fas_user( + trac_ticket['owner']) pagure_issue_user = self.fas.find_fas_user(trac_ticket['reporter']) if not pagure_issue_user.name: - pagure_issue_user = User(name=trac_ticket['reporter'], - fullname=trac_ticket['reporter'], - emails=[trac_ticket['reporter']+'@fedoraproject.org']) + pagure_issue_user = User( + name=trac_ticket['reporter'], + fullname=trac_ticket['reporter'], + emails=[trac_ticket['reporter']+'@fedoraproject.org']) else: pagure_issue_assignee = User(name='', fullname='', emails=[]) - pagure_issue_user = User(name=trac_ticket['reporter'], - fullname=trac_ticket['reporter'], - emails=[trac_ticket['reporter']+'@fedoraproject.org']) + pagure_issue_user = User( + name=trac_ticket['reporter'], + fullname=trac_ticket['reporter'], + emails=[trac_ticket['reporter']+'@fedoraproject.org']) pagure_issue_tags = [] if self.tags: - pagure_issue_tags = filter(lambda x: x != '', trac_ticket['keywords'].split(' ')) + pagure_issue_tags = filter( + lambda x: x != '', trac_ticket['keywords'].split(' ')) if trac_ticket['milestone'] != '': pagure_issue_tags.append(str(trac_ticket['milestone']))