From a6c253646b80a70cd07799585dba0d19d269cad0 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Apr 15 2016 19:26:00 +0000 Subject: [PATCH 1/24] trac importer skeleton --- diff --git a/pagure_importer/lib/sources/importer_trac.py b/pagure_importer/lib/sources/importer_trac.py new file mode 100644 index 0000000..3ff00a7 --- /dev/null +++ b/pagure_importer/lib/sources/importer_trac.py @@ -0,0 +1,106 @@ +from xmlrpclib import ServerProxy +import pagure_importer +import pagure_importer.lib +from pagure_importer.lib import models +from datetime import datetime + +class TracImporter(): + '''Pagure importer for trac instance''' + + def __init__(self, trac_project_url): + self.trac = ServerProxy(trac_project_url + '/rpc') + + def import_issues(self, repo_path, repo_folder, trac_query='report=9&order=id'): + '''Import issues from trac instance using xmlrpc API''' + tickets_list = [] + tickets_id = self.trac.ticket.query(trac_query) + + for ticket_id in tickets_id: + + trac_ticket = self.trac.ticket.get(ticket_id)[3] + pagure_issue_title = trac_ticket['summary'] + pagure_issue_content = trac_ticket['description'] + + if pagure_issue_content == '': + pagure_issue_content = '#No Description Provided' + + if trac_ticket['status'] != 'closed': + pagure_issue_status = 'Open' + else: + pagure_issue_status = 'Fixed' + + pagure_issue_created_at = datetime.strptime(self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") + + pagure_issue_assignee = trac_ticket['owner'] + + # if github_issue.labels: + # pagure_issue_tags = [i.name for i in github_issue.labels] + # else: + pagure_issue_tags = [] + + pagure_issue_depends = [] + pagure_issue_blocks = [] + pagure_issue_is_private = False + + pagure_issue_user = models.User( + name=trac_ticket['reporter'], + fullname=None, + emails=[None]) + + pagure_issue = models.Issue( + id=None, + title=pagure_issue_title, + content=pagure_issue_content, + status=pagure_issue_status, + date_created=pagure_issue_created_at, + user=pagure_issue_user.to_json(), + private=pagure_issue_is_private, + tags=pagure_issue_tags, + depends=pagure_issue_depends, + blocks=pagure_issue_blocks, + assignee=pagure_issue_assignee) + + pagure_issue_comments = self.trac.ticket.changeLog(ticket_id) + comments = [] + for comment in pagure_issue_comments: + if comment[2] == 'comment' and comment[4] != '': + comment_user = comment[1] + pagure_issue_comment_user_email = None + pagure_issue_comment_body = comment[4] + pagure_issue_comment_created_at = datetime.strptime(comment[0].value, "%Y%m%dT%H:%M:%S") + pagure_issue_comment_updated_at = None + + # No idea what to do with this right now + # editor: not supported by github api + pagure_issue_comment_parent = None + pagure_issue_comment_editor = None + + # comment updated at + pagure_issue_comment_edited_on = None + + # The User who commented + pagure_issue_comment_user = models.User( + name=comment[1], + fullname=None, + emails=None) + + # Object to represent comment on an issue + pagure_issue_comment = models.IssueComment( + id=None, + comment=pagure_issue_comment_body, + parent=pagure_issue_comment_parent, + date_created=pagure_issue_comment_created_at, + user=pagure_issue_comment_user.to_json(), + edited_on=pagure_issue_comment_edited_on, + editor=pagure_issue_comment_editor) + + comments.append(pagure_issue_comment.to_json()) + + # add all the comments to the issue object + pagure_issue.comments = comments + + # update the local git repo + print 'Update repo with issue :' + str(ticket_id) + pagure_importer.lib.git.update_git(pagure_issue, + repo_path, + repo_folder) diff --git a/pagure_importer/run.py b/pagure_importer/run.py index 2753298..d327228 100644 --- a/pagure_importer/run.py +++ b/pagure_importer/run.py @@ -6,9 +6,11 @@ import pagure_importer import pagure_importer.lib import pagure_importer.lib.sources from pagure_importer.lib.sources.importer_github import GithubImporter +from pagure_importer.lib.sources.importer_trac import TracImporter from pagure_importer.lib import generate_json_for_github_contributors, \ - generate_json_for_github_issue_commentors, \ - assemble_github_contributors_commentors + generate_json_for_github_issue_commentors, \ + assemble_github_contributors_commentors + def github_handler(item): if item.lower() == 'issues': @@ -16,22 +18,30 @@ def github_handler(item): gen_json = raw_input('Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') if gen_json == 'n': github_importer = GithubImporter( - github_username=github_username, - github_password=github_password, - github_project_name=github_project_name) + github_username=github_username, + github_password=github_password, + github_project_name=github_project_name) github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) else: generate_json_for_github_contributors( - github_username, - github_password, - github_project_name) + github_username, + github_password, + github_project_name) generate_json_for_github_issue_commentors( - github_username, - github_password, - github_project_name) + github_username, + github_password, + github_project_name) assemble_github_contributors_commentors() return + +def trac_handler(item, fedora=False): + if item.lower() == 'issues': + trac_url = raw_input('Enter the trac project url: ') + trac_importer = TracImporter(trac_url) + trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) + + def main(): source = raw_input('Enter source from where you want to import: ') if source.lower() not in IMPORT_SOURCES: @@ -45,6 +55,8 @@ def main(): if source.lower() == 'github': github_handler(item) + elif source.lower() == 'fedorahosted': + trac_handler(item, fedora=True) return if __name__ == '__main__': diff --git a/pagure_importer/settings.py b/pagure_importer/settings.py index 9761aed..077ea40 100644 --- a/pagure_importer/settings.py +++ b/pagure_importer/settings.py @@ -1,7 +1,7 @@ import os -IMPORT_SOURCES = ['github'] -IMPORT_OPTIONS = {'github': ['issues']} +IMPORT_SOURCES = ['github', 'fedorahosted'] +IMPORT_OPTIONS = {'github': ['issues'], 'fedorahosted': ['issues']} REPO_NAME = os.environ.get('REPO_NAME', None) #this has to be a bare repo REPO_PATH = os.environ.get('REPO_PATH', None) #the parent of the git directory From b51b561f8ffadae50c804df7a0b43620a3617867 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Apr 16 2016 21:07:24 +0000 Subject: [PATCH 2/24] Refactor trac importer for more flexibility --- diff --git a/pagure_importer/lib/sources/importer_trac.py b/pagure_importer/lib/sources/importer_trac.py index 3ff00a7..2e6fce2 100644 --- a/pagure_importer/lib/sources/importer_trac.py +++ b/pagure_importer/lib/sources/importer_trac.py @@ -2,104 +2,132 @@ from xmlrpclib import ServerProxy import pagure_importer import pagure_importer.lib from pagure_importer.lib import models +from fedora.client.fas2 import AccountSystem from datetime import datetime + class TracImporter(): '''Pagure importer for trac instance''' def __init__(self, trac_project_url): self.trac = ServerProxy(trac_project_url + '/rpc') - - def import_issues(self, repo_path, repo_folder, trac_query='report=9&order=id'): + fas_url = 'https://admin.fedoraproject.org/accounts' + fas_username = 'user' + fas_password = 'pass' + self.fasclient = AccountSystem(fas_url, username=fas_username, + password=fas_password) + + def _find_fas_user(self, user): + person = self.fasclient.person_by_username(user) + human_name = person['human_name'] + email = person['email'] + pagure_user = models.User( + name=user, + fullname=human_name, + emails=[email]) + return pagure_user + + def _get_ticket_tags(self, trac_ticket): + return [] + + def _get_ticket_status(self, trac_ticket): + ''' Converts Trac ticket status + to Pagure issue status''' + + if trac_ticket['status'] != 'closed': + ticket_status = 'Open' + else: + ticket_status = 'Fixed' + return ticket_status + + def _populate_comments(self, trac_comments): + comments = [] + for comment in trac_comments: + if comment[2] == 'comment' and comment[4] != '': + comment_user = comment[1] + pagure_issue_comment_user_email = None + pagure_issue_comment_body = comment[4] + pagure_issue_comment_created_at = datetime.strptime( + comment[0].value, "%Y%m%dT%H:%M:%S") + pagure_issue_comment_updated_at = None + + # No idea what to do with this right now + # editor: not supported by github api + pagure_issue_comment_parent = None + pagure_issue_comment_editor = None + + # comment updated at + pagure_issue_comment_edited_on = None + + # The User who commented + pagure_issue_comment_user = self._find_fas_user(comment[1]) + + # Object to represent comment on an issue + pagure_issue_comment = models.IssueComment( + id=None, + comment=pagure_issue_comment_body, + parent=pagure_issue_comment_parent, + date_created=pagure_issue_comment_created_at, + user=pagure_issue_comment_user.to_json(), + edited_on=pagure_issue_comment_edited_on, + editor=pagure_issue_comment_editor) + + comments.append(pagure_issue_comment.to_json()) + return comments + + def _populate_issue(self, ticket_id): + trac_ticket = self.trac.ticket.get(ticket_id)[3] + pagure_issue_title = trac_ticket['summary'] + pagure_issue_content = trac_ticket['description'] + + if pagure_issue_content == '': + pagure_issue_content = '#No Description Provided' + + pagure_issue_status = self._get_ticket_status(trac_ticket) + + pagure_issue_created_at = datetime.strptime( + self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") + + pagure_issue_assignee = trac_ticket['owner'] + + pagure_issue_tags = self._get_ticket_tags(trac_ticket) + + pagure_issue_depends = [] + pagure_issue_blocks = [] + pagure_issue_is_private = False + + pagure_issue_user = self._find_fas_user(trac_ticket['reporter']) + + pagure_issue = models.Issue( + id=None, + title=pagure_issue_title, + content=pagure_issue_content, + status=pagure_issue_status, + date_created=pagure_issue_created_at, + user=pagure_issue_user.to_json(), + private=pagure_issue_is_private, + tags=pagure_issue_tags, + depends=pagure_issue_depends, + blocks=pagure_issue_blocks, + assignee=pagure_issue_assignee) + return pagure_issue + + def import_issues(self, repo_path, repo_folder, + trac_query='max=0&order=id'): '''Import issues from trac instance using xmlrpc API''' - tickets_list = [] tickets_id = self.trac.ticket.query(trac_query) for ticket_id in tickets_id: - trac_ticket = self.trac.ticket.get(ticket_id)[3] - pagure_issue_title = trac_ticket['summary'] - pagure_issue_content = trac_ticket['description'] - - if pagure_issue_content == '': - pagure_issue_content = '#No Description Provided' - - if trac_ticket['status'] != 'closed': - pagure_issue_status = 'Open' - else: - pagure_issue_status = 'Fixed' - - pagure_issue_created_at = datetime.strptime(self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") - - pagure_issue_assignee = trac_ticket['owner'] - - # if github_issue.labels: - # pagure_issue_tags = [i.name for i in github_issue.labels] - # else: - pagure_issue_tags = [] - - pagure_issue_depends = [] - pagure_issue_blocks = [] - pagure_issue_is_private = False - - pagure_issue_user = models.User( - name=trac_ticket['reporter'], - fullname=None, - emails=[None]) - - pagure_issue = models.Issue( - id=None, - title=pagure_issue_title, - content=pagure_issue_content, - status=pagure_issue_status, - date_created=pagure_issue_created_at, - user=pagure_issue_user.to_json(), - private=pagure_issue_is_private, - tags=pagure_issue_tags, - depends=pagure_issue_depends, - blocks=pagure_issue_blocks, - assignee=pagure_issue_assignee) + pagure_issue = self._populate_issue(ticket_id) pagure_issue_comments = self.trac.ticket.changeLog(ticket_id) - comments = [] - for comment in pagure_issue_comments: - if comment[2] == 'comment' and comment[4] != '': - comment_user = comment[1] - pagure_issue_comment_user_email = None - pagure_issue_comment_body = comment[4] - pagure_issue_comment_created_at = datetime.strptime(comment[0].value, "%Y%m%dT%H:%M:%S") - pagure_issue_comment_updated_at = None - - # No idea what to do with this right now - # editor: not supported by github api - pagure_issue_comment_parent = None - pagure_issue_comment_editor = None - - # comment updated at - pagure_issue_comment_edited_on = None - - # The User who commented - pagure_issue_comment_user = models.User( - name=comment[1], - fullname=None, - emails=None) - - # Object to represent comment on an issue - pagure_issue_comment = models.IssueComment( - id=None, - comment=pagure_issue_comment_body, - parent=pagure_issue_comment_parent, - date_created=pagure_issue_comment_created_at, - user=pagure_issue_comment_user.to_json(), - edited_on=pagure_issue_comment_edited_on, - editor=pagure_issue_comment_editor) - - comments.append(pagure_issue_comment.to_json()) - - # add all the comments to the issue object - pagure_issue.comments = comments - - # update the local git repo + comments = self._populate_comments(pagure_issue_comments) + + # add all the comments to the issue object + pagure_issue.comments = comments + + # update the local git repo print 'Update repo with issue :' + str(ticket_id) pagure_importer.lib.git.update_git(pagure_issue, repo_path, From 404bfcc5e365dbc1dd26fe3b0ae71f53387f81df Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Apr 22 2016 08:39:24 +0000 Subject: [PATCH 3/24] gitignore changes --- diff --git a/.gitignore b/.gitignore index ac43ebc..358cdf2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,11 @@ +dist/ +build/ *.pyc *.py~ +*.pyo +*.swp +*.egg-info +*.conf +.coverage +*.json +*.csv From 0cfa26bc59668717d954f5673438337597f671ee Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Apr 22 2016 08:45:09 +0000 Subject: [PATCH 4/24] FAS and TRAC support added to importer --- diff --git a/pagure_importer/lib/fas.py b/pagure_importer/lib/fas.py new file mode 100644 index 0000000..b63f665 --- /dev/null +++ b/pagure_importer/lib/fas.py @@ -0,0 +1,23 @@ +from fedora.client.fas2 import AccountSystem +from pagure_importer.lib.models import User + + +class FASclient (): + def __init__(self, fas_username, fas_password, fas_url): + self.fasclient = AccountSystem(fas_url, username=fas_username, + password=fas_password) + + anonymous = User(name='', fullname='', emails=[]) + self.fasuser = {'': anonymous} + + def find_fas_user(self, user): + + if user not in self.fasuser.keys(): + person = self.fasclient.person_by_username(user) + if not person: + return self.fasuser[''] + + self.fasuser[user] = User(name=user, + fullname=person['human_name'], + emails=[person['email']]) + return self.fasuser[user] diff --git a/pagure_importer/lib/sources/importer_trac.py b/pagure_importer/lib/sources/importer_trac.py index 2e6fce2..f69a3ce 100644 --- a/pagure_importer/lib/sources/importer_trac.py +++ b/pagure_importer/lib/sources/importer_trac.py @@ -1,21 +1,17 @@ from xmlrpclib import ServerProxy import pagure_importer import pagure_importer.lib -from pagure_importer.lib import models -from fedora.client.fas2 import AccountSystem -from datetime import datetime +from pagure_importer.lib.fas import FASclient +from pagure_importer.lib import trac class TracImporter(): '''Pagure importer for trac instance''' def __init__(self, trac_project_url): - self.trac = ServerProxy(trac_project_url + '/rpc') - fas_url = 'https://admin.fedoraproject.org/accounts' - fas_username = 'user' - fas_password = 'pass' - self.fasclient = AccountSystem(fas_url, username=fas_username, - password=fas_password) + self.tracclient = ServerProxy(trac_project_url + '/rpc') + self.fasclient = FASclient('user', 'password', + 'https://admin.fedoraproject.org/accounts') def _find_fas_user(self, user): person = self.fasclient.person_by_username(user) @@ -115,14 +111,16 @@ class TracImporter(): def import_issues(self, repo_path, repo_folder, trac_query='max=0&order=id'): '''Import issues from trac instance using xmlrpc API''' - tickets_id = self.trac.ticket.query(trac_query) + tickets_id = self.tracclient.ticket.query(trac_query) for ticket_id in tickets_id: - pagure_issue = self._populate_issue(ticket_id) + pagure_issue = trac.populate_issue(self.tracclient, + self.fasclient, ticket_id) - pagure_issue_comments = self.trac.ticket.changeLog(ticket_id) - comments = self._populate_comments(pagure_issue_comments) + pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) + comments = trac.populate_comments(self.fasclient, + pagure_issue_comments) # add all the comments to the issue object pagure_issue.comments = comments diff --git a/pagure_importer/lib/trac.py b/pagure_importer/lib/trac.py new file mode 100644 index 0000000..fb3d062 --- /dev/null +++ b/pagure_importer/lib/trac.py @@ -0,0 +1,90 @@ +from pagure_importer.lib.models import IssueComment, Issue +from datetime import datetime + + +def get_ticket_tags(trac_ticket): + return [] + + +def get_ticket_status(trac_ticket): + ''' Converts Trac ticket status + to Pagure issue status''' + + if trac_ticket['status'] != 'closed': + ticket_status = 'Open' + else: + ticket_status = 'Fixed' + return ticket_status + + +def populate_comments(fasclient, trac_comments): + comments = [] + for comment in trac_comments: + if comment[2] == 'comment' and comment[4] != '': + comment_user = comment[1] + pagure_issue_comment_user_email = None + pagure_issue_comment_body = comment[4] + pagure_issue_comment_created_at = datetime.strptime( + comment[0].value, "%Y%m%dT%H:%M:%S") + pagure_issue_comment_updated_at = None + + # No idea what to do with this right now + # editor: not supported by github api + pagure_issue_comment_parent = None + pagure_issue_comment_editor = None + + # comment updated at + pagure_issue_comment_edited_on = None + + # The User who commented + pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) + + # Object to represent comment on an issue + pagure_issue_comment = IssueComment( + id=None, + comment=pagure_issue_comment_body, + parent=pagure_issue_comment_parent, + date_created=pagure_issue_comment_created_at, + user=pagure_issue_comment_user.to_json(), + edited_on=pagure_issue_comment_edited_on, + editor=pagure_issue_comment_editor) + + comments.append(pagure_issue_comment.to_json()) + return comments + + +def populate_issue(trac, fasclient, ticket_id): + trac_ticket = trac.ticket.get(ticket_id)[3] + pagure_issue_title = trac_ticket['summary'] + pagure_issue_content = trac_ticket['description'] + + if pagure_issue_content == '': + pagure_issue_content = '#No Description Provided' + + pagure_issue_status = get_ticket_status(trac_ticket) + + pagure_issue_created_at = datetime.strptime( + trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") + + pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) + + pagure_issue_tags = get_ticket_tags(trac_ticket) + + pagure_issue_depends = [] + pagure_issue_blocks = [] + pagure_issue_is_private = False + + pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) + pagure_issue = Issue( + id=ticket_id, + title=pagure_issue_title, + content=pagure_issue_content, + status=pagure_issue_status, + date_created=pagure_issue_created_at, + user=pagure_issue_user.to_json(), + private=pagure_issue_is_private, + tags=pagure_issue_tags, + depends=pagure_issue_depends, + blocks=pagure_issue_blocks, + assignee=pagure_issue_assignee.to_json()) + return pagure_issue From 84add1511833031f88b5c16f9c2b6796f37be456 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 04 2016 19:13:36 +0000 Subject: [PATCH 5/24] Reworked app structure to use click cli framework. App structure base on https://pagure.io/pag --- diff --git a/diff b/diff new file mode 100644 index 0000000..2a92e6e --- /dev/null +++ b/diff @@ -0,0 +1,1922 @@ +diff --git a/pagure_importer/__init__.py b/pagure_importer/__init__.py +index fbab797..8b13789 100644 +--- a/pagure_importer/__init__.py ++++ b/pagure_importer/__init__.py +@@ -1,2 +1 @@ +-import lib +-import settings ++ +diff --git a/pagure_importer/app.py b/pagure_importer/app.py +new file mode 100644 +index 0000000..6dc3277 +--- /dev/null ++++ b/pagure_importer/app.py +@@ -0,0 +1,23 @@ ++#!/usr/bin/env python ++ ++import click ++import os ++ ++REPO_NAME = os.environ.get('REPO_NAME', None) # this has to be a bare repo ++REPO_PATH = os.environ.get('REPO_PATH', None) # the parent of the git directory ++ ++ ++@click.group() ++def app(): ++ pass ++ ++__all__ = [ ++ 'app', ++] ++ ++# from .commands import github ++from .commands import fedorahosted ++from .commands import github ++ ++if __name__ == '__main__': ++ app() +diff --git a/pagure_importer/commands/__init__.py b/pagure_importer/commands/__init__.py +new file mode 100644 +index 0000000..e69de29 +diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py +new file mode 100644 +index 0000000..650d133 +--- /dev/null ++++ b/pagure_importer/commands/fedorahosted.py +@@ -0,0 +1,15 @@ ++import click ++import getpass ++from pagure_importer.app import app, REPO_NAME, REPO_PATH ++from pagure_importer.utils import importer_trac ++from pagure_importer.utils.fas import FASclient ++ ++@app.command() ++@click.argument('project_url') ++def fedorahosted(project_url): ++ fas_username = raw_input('Enter you Github Username: ') ++ fas_password = getpass.getpass('Enter your github password: ') ++ fasclient = FASclient(fas_username, fas_password, ++ 'https://admin.fedoraproject.org/accounts') ++ trac_importer = importer_trac.TracImporter(project_url, fasclient) ++ trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) +diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py +new file mode 100644 +index 0000000..26ae4fa +--- /dev/null ++++ b/pagure_importer/commands/github.py +@@ -0,0 +1,40 @@ ++import click ++import getpass ++ ++from pagure_importer.app import app, REPO_NAME, REPO_PATH ++from pagure_importer.utils.importer_github import GithubImporter ++from pagure_importer.utils import ( ++ generate_json_for_github_contributors, ++ generate_json_for_github_issue_commentors, ++ assemble_github_contributors_commentors ++) ++ ++ ++def form_github_issues(): ++ github_username = raw_input('Enter you Github Username: ') ++ github_password = getpass.getpass('Enter your github password: ') ++ github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') ++ return (github_username, github_password, github_project_name) ++ ++@app.command() ++def github(): ++ github_username, github_password, github_project_name = form_github_issues() ++ gen_json = raw_input( ++ 'Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') ++ if gen_json == 'n': ++ github_importer = GithubImporter( ++ github_username=github_username, ++ github_password=github_password, ++ github_project_name=github_project_name) ++ github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) ++ else: ++ generate_json_for_github_contributors( ++ github_username, ++ github_password, ++ github_project_name) ++ generate_json_for_github_issue_commentors( ++ github_username, ++ github_password, ++ github_project_name) ++ assemble_github_contributors_commentors() ++ return +diff --git a/pagure_importer/forms.py b/pagure_importer/forms.py +deleted file mode 100644 +index a111114..0000000 +--- a/pagure_importer/forms.py ++++ /dev/null +@@ -1,10 +0,0 @@ +-import getpass +- +-from lib.sources.importer_github import GithubImporter +-from settings import REPO_PATH, REPO_NAME +- +-def form_github_issues(): +- github_username = raw_input('Enter you Github Username: ') +- github_password = getpass.getpass('Enter your github password: ') +- github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') +- return (github_username, github_password, github_project_name) +diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py +deleted file mode 100644 +index 9a64e3b..0000000 +--- a/pagure_importer/lib/__init__.py ++++ /dev/null +@@ -1,170 +0,0 @@ +-import git +-import models +- +-import csv +-import os +-import getpass +-import requests +-import json +-from github import Github +-from requests.auth import HTTPBasicAuth +-import pagure_importer +-import pagure_importer.lib +-from pagure_importer.lib.exceptions import FileNotFound, EmailNotFound +- +-def generate_json_for_github_contributors(github_username, github_password, \ +- github_project_name): +- ''' Creates a file containing a list of dicts containing the username and +- emails of the contributors in the given github project +- ''' +- +- github_obj = Github(github_username, github_password) +- project = github_obj.get_repo(github_project_name) +- commits_url = project.commits_url.replace('{/sha}', '') +- +- page = 0 +- contributors = [] +- while True: +- page += 1 +- payload = {'page': page } +- data_ = json.loads(requests.get(commits_url, params=payload, +- auth=HTTPBasicAuth(github_username, github_password)).text) +- +- if not data_: +- break +- +- for data in data_: +- try: +- contributor = data['commit']['committer'] +- contributor_email = contributor['email'] +- contributor_fullname = contributor['name'] +- contributor_name = data['committer']['login'] +- except TypeError: +- print 'Maybe one of the contributors is dropped because of lack of details' +- continue +- +- json_data = { +- 'name': contributor_name, +- 'fullname': contributor_fullname, +- 'emails': [contributor_email] +- } +- +- present = False +- for i in contributors: +- if i == json_data: +- present = True +- break +- +- if not present: +- print 'contributor added: ', contributor_name +- contributors.append(json_data) +- +- with open('contributors.json', 'w') as f: +- f.write(json.dumps(contributors)) +- +- return +- +- +-def generate_json_for_github_issue_commentors(github_username, github_password, \ +- github_project_name): +- ''' Will create a json file containing details of all the user +- who have commented on any issue in the given project +- ''' +- +- github_obj = Github(github_username, github_password) +- project = github_obj.get_repo(github_project_name) +- issue_comment_url = project.issue_comment_url.replace('{/number}', '') +- +- page = 0 +- issue_commentors = [] +- while True: +- page += 1 +- payload = {'page': page } +- data_ = json.loads(requests.get(issue_comment_url, params=payload, +- auth=HTTPBasicAuth(github_username, github_password)).text) +- +- if not data_: +- break +- +- for data in data_: +- try: +- commentor = data['user']['login'] +- except TypeError: +- print 'Maybe one of the issue commentors have been dropped because of lack of details' +- continue +- +- present = False +- for i in issue_commentors: +- if i == commentor: +- present = True +- break +- +- if not present: +- print 'commentor added: ', commentor +- issue_commentors.append(commentor) +- +- with open('issue_commentors.json', 'w') as f: +- f.write(json.dumps(issue_commentors)) +- return +- +- +-def assemble_github_contributors_commentors(): +- ''' It uses the files: issue_commentors.json and contributors.json +- Assembles and creates a file: assembled_commentors.csv +- To use: just fill the empty blocks under emails column''' +- +- with open('issue_commentors.json', 'r') as ic: +- issue_names = json.load(ic) +- +- with open('contributors.json', 'r') as c: +- contributors = json.load(c) +- +- names = [] +- for i in issue_names: +- found = False +- for j in contributors: +- if j.get('name', None) == i: +- j['emails'] = j.get('emails')[0] +- names.append(j) +- found = True +- +- if not found: +- d = {'name': i, 'fullname': None, 'emails': None} +- names.append(d) +- +- with open('assembled_commentors.csv', 'w') as ac: +- field_names = ['name', 'fullname', 'emails'] +- writer = csv.DictWriter(ac, fieldnames=field_names) +- +- writer.writeheader() +- for name in names: +- writer.writerow(name) +- +- +-def github_get_commentor_email(name): +- ''' Will return the issue commentor email as given in the +- assembled_commentors.csv file +- ''' +- +- if not os.path.exists('assembled_commentors.csv'): +- raise FileNotFound('The assembled_commentors.json file must be present \ +- Rerun the program and choose to generate the json files') +- +- data = [] +- with open('assembled_commentors.csv') as ac: +- reader = csv.DictReader(ac) +- for row in reader: +- data.append(dict( \ +- (('name', row['name']), \ +- ('fullname', row['fullname']), \ +- ('emails', row['emails'])))) +- +- +- for i in data: +- if i.get('name', None) == name: +- if i['emails']: +- return str(i['emails']) +- else: +- raise EmailNotFound('You need to fill out all the emails of the \ +- issue commentors') +- +diff --git a/pagure_importer/lib/exceptions.py b/pagure_importer/lib/exceptions.py +deleted file mode 100644 +index a5ab5fe..0000000 +--- a/pagure_importer/lib/exceptions.py ++++ /dev/null +@@ -1,22 +0,0 @@ +-class GithubBadCredentials(Exception): +- ''' Raised when username/password for github is wrong ''' +- def __init__(self, msg): +- self.msg = msg +- +- +-class GithubRepoNotFound(Exception): +- ''' Raised when the repo is not found for the user ''' +- def __init__(self, msg): +- self.msg = msg +- +- +-class FileNotFound(Exception): +- ''' Raised when a certain file is not found ''' +- def __init__(self, msg): +- self.msg = msg +- +- +-class EmailNotFound(Exception): +- ''' Raised when email is not found ''' +- def __init__(self, msg): +- self.msg = msg +diff --git a/pagure_importer/lib/fas.py b/pagure_importer/lib/fas.py +deleted file mode 100644 +index b63f665..0000000 +--- a/pagure_importer/lib/fas.py ++++ /dev/null +@@ -1,23 +0,0 @@ +-from fedora.client.fas2 import AccountSystem +-from pagure_importer.lib.models import User +- +- +-class FASclient (): +- def __init__(self, fas_username, fas_password, fas_url): +- self.fasclient = AccountSystem(fas_url, username=fas_username, +- password=fas_password) +- +- anonymous = User(name='', fullname='', emails=[]) +- self.fasuser = {'': anonymous} +- +- def find_fas_user(self, user): +- +- if user not in self.fasuser.keys(): +- person = self.fasclient.person_by_username(user) +- if not person: +- return self.fasuser[''] +- +- self.fasuser[user] = User(name=user, +- fullname=person['human_name'], +- emails=[person['email']]) +- return self.fasuser[user] +diff --git a/pagure_importer/lib/git.py b/pagure_importer/lib/git.py +deleted file mode 100644 +index 33eba48..0000000 +--- a/pagure_importer/lib/git.py ++++ /dev/null +@@ -1,98 +0,0 @@ +-''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py +- by pingou@pingoured.fr +-''' +- +-import shutil +-import os +-import pygit2 +-import tempfile +-import json +- +-from repo import * +- +-def update_git(obj, repo_path, repo_folder): +- """ Update the given issue in its git. +- This method forks the provided repo, add/edit the issue whose file name +- is defined by the uid field of the issue and if there are additions/ +- changes commit them and push them back to the original repo. +- """ +- +- if not repo_folder: +- return +- +- # Get the fork +- repopath = os.path.join(repo_folder, repo_path) +- +- # Clone the repo into a temp folder +- newpath = tempfile.mkdtemp(prefix='pagure-') +- new_repo = pygit2.clone_repository(repopath, newpath) +- +- file_path = os.path.join(newpath, obj.uid) +- +- # Get the current index +- index = new_repo.index +- +- # Are we adding files +- added = False +- if not os.path.exists(file_path): +- added = True +- +- # Write down what changed +- with open(file_path, 'w') as stream: +- stream.write(json.dumps( +- obj.to_json(), sort_keys=True, indent=4, +- separators=(',', ': '))) +- +- # Retrieve the list of files that changed +- diff = new_repo.diff() +- files = [] +- for p in diff: +- if hasattr(p, 'new_file_path'): +- files.append(p.new_file_path) +- elif hasattr(p, 'delta'): +- files.append(p.delta.new_file.path) +- +- # Add the changes to the index +- if added: +- index.add(obj.uid) +- for filename in files: +- index.add(filename) +- +- # If not change, return +- if not files and not added: +- shutil.rmtree(newpath) +- return +- +- # See if there is a parent to this commit +- parent = None +- try: +- parent = new_repo.head.get_object().oid +- except pygit2.GitError: +- pass +- +- parents = [] +- if parent: +- parents.append(parent) +- +- # Author/commiter will always be this one +- author = pygit2.Signature(name='pagure', email='pagure') +- +- # Actually commit +- new_repo.create_commit( +- 'refs/heads/master', +- author, +- author, +- 'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title), +- new_repo.index.write_tree(), +- parents) +- index.write() +- +- # Push to origin +- ori_remote = new_repo.remotes[0] +- master_ref = new_repo.lookup_reference('HEAD').resolve() +- refname = '%s:%s' % (master_ref.name, master_ref.name) +- +- PagureRepo.push(ori_remote, refname) +- +- # Remove the clone +- shutil.rmtree(newpath) +diff --git a/pagure_importer/lib/models.py b/pagure_importer/lib/models.py +deleted file mode 100644 +index 3aa3d83..0000000 +--- a/pagure_importer/lib/models.py ++++ /dev/null +@@ -1,105 +0,0 @@ +-# -*- coding: utf-8 -*- +- +-import datetime +-import json +-import uuid +- +-class Issue(): +- ''' Represents an Issue ''' +- +- def __init__( +- self, id, title, content, +- status, date_created, user, private, tags, +- depends, blocks, assignee, comments=None): +- +- self.id = id +- self.title = title +- self.content = content +- self.status = status +- self.date_created = date_created +- self.user = user +- self.private = private +- self.tags = tags +- self.depends = depends +- self.blocks = blocks +- self.assignee = assignee +- self.comments = comments +- self.uid = uuid.uuid4().hex +- +- def to_json(self): +- ''' Returns a dictionary representation of the issue. +- +- ''' +- output = { +- 'id': self.id, +- 'title': self.title, +- 'content': self.content, +- 'status': self.status, +- 'date_created': self.date_created.strftime('%s'), +- 'user': self.user, +- 'private': self.private, +- 'tags': self.tags, +- 'depends': self.depends, +- 'blocks': self.blocks, +- 'assignee': self.assignee, +- 'comments': self.comments +- } +- +- return output +- +- @property +- def isa(self): +- return 'issue' +- +- +-class IssueComment(): +- ''' Represent a comment for an issue ''' +- +- def __init__( +- self, id, comment, date_created, +- user, parent=None, edited_on=None, editor=None): +- +- self.id = id +- self.comment = comment +- self.parent = parent +- self.date_created = date_created +- self.user = user +- self.edited_on = edited_on +- self.editor = editor +- +- def to_json(self): +- ''' Returns a dictionary representation of the issue. ''' +- +- output = { +- 'id': self.id, +- 'comment': self.comment, +- 'parent': self.parent, +- 'date_created': self.date_created.strftime('%s'), +- 'user': self.user, +- 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, +- 'editor': self.editor or None +- } +- +- return output +- +- +-class User(): +- ''' Represents a User ''' +- +- def __init__( +- self, name, emails, +- fullname=None): +- self.name = name +- self.fullname = fullname +- self.emails = emails +- +- def to_json(self): +- ''' Return a representation of the User in a dictionary. ''' +- +- output = { +- 'name': self.name, +- 'fullname': self.fullname, +- 'emails': self.emails +- } +- +- return output +diff --git a/pagure_importer/lib/repo.py b/pagure_importer/lib/repo.py +deleted file mode 100644 +index f46b31a..0000000 +--- a/pagure_importer/lib/repo.py ++++ /dev/null +@@ -1,70 +0,0 @@ +-# -*- coding: utf-8 -*- +- +-''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py +- by pingou@pingoured.fr +-''' +- +- +-import pygit2 +-import sys +- +- +-def get_pygit2_version(): +- ''' Return pygit2 version as a tuple of integers. +- This is needed for correct version comparison. +- ''' +- return tuple([int(i) for i in pygit2.__version__.split('.')]) +- +- +-class PagureRepo(pygit2.Repository): +- """ An utility class allowing to go around pygit2's inability to be +- stable. +- +- """ +- +- @staticmethod +- def push(remote, refname): +- """ Push the given reference to the specified remote. """ +- pygit2_version = get_pygit2_version() +- if pygit2_version >= (0, 22): +- remote.push([refname]) +- else: +- remote.push(refname) +- +- def pull(self, remote_name='origin', branch='master', force=False): +- ''' pull changes for the specified remote (defaults to origin). +- +- Code from MichaelBoselowitz at: +- https://github.com/MichaelBoselowitz/pygit2-examples/blob/ +- 68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58 +- licensed under the MIT license. +- ''' +- +- for remote in self.remotes: +- if remote.name == remote_name: +- remote.fetch() +- remote_master_id = self.lookup_reference( +- 'refs/remotes/origin/%s' % branch).target +- +- if force: +- repo_branch = self.lookup_reference( +- 'refs/heads/%s' % branch) +- repo_branch.set_target(remote_master_id) +- +- merge_result, _ = self.merge_analysis(remote_master_id) +- # Up to date, do nothing +- if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: +- return +- # We can just fastforward +- elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: +- self.checkout_tree(self.get(remote_master_id)) +- master_ref = self.lookup_reference( +- 'refs/heads/%s' % branch) +- master_ref.set_target(remote_master_id) +- self.head.set_target(remote_master_id) +- elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: +- sys.exit('Pulling remote changes leads to a conflict') +- else: +- print 'Unexpected merge result: %s' % ( +- pygit2.GIT_MERGE_ANALYSIS_NORMAL) +- raise AssertionError('Unknown merge analysis result') +diff --git a/pagure_importer/lib/sources/__init__.py b/pagure_importer/lib/sources/__init__.py +deleted file mode 100644 +index e69de29..0000000 +diff --git a/pagure_importer/lib/sources/importer_github.py b/pagure_importer/lib/sources/importer_github.py +deleted file mode 100644 +index 2859e99..0000000 +--- a/pagure_importer/lib/sources/importer_github.py ++++ /dev/null +@@ -1,134 +0,0 @@ +-from github import Github +-import pagure_importer +-import pagure_importer.lib +-from pagure_importer.lib import models +-from pagure_importer.lib import github_get_commentor_email +-from pagure_importer.lib.exceptions import GithubBadCredentials, GithubRepoNotFound +- +-class GithubImporter(): +- ''' Imports from Github using PyGithub and libpagure ''' +- def __init__( +- self, +- github_username, +- github_password, +- github_project_name): +- self.github_username = github_username +- self.github_password = github_password +- self.github_project_name = github_project_name +- self.github = Github(github_username, github_password) +- +- def import_issues(self, repo_path, repo_folder, status='all'): +- ''' Imports the issues on github for +- the given project +- ''' +- github_user = None +- try: +- github_user = self.github.get_user(self.github_username) +- except: +- raise GithubBadCredentials( +- 'Given github credentials are not correct') +- repo = self.github.get_repo(self.github_project_name) +- try: +- repo_name = repo.name +- except: +- raise GithubRepoNotFound( +- 'Repo not found, project name wrong') +- +- for github_issue in repo.get_issues(state=status): +- +- #title of the issue +- pagure_issue_title = github_issue.title +- +- #body of the issue +- if github_issue.body: +- pagure_issue_content = github_issue.body +- else: +- pagure_issue_content = '#No Description Provided' +- +- #Some details of a issue +- if github_issue.state != 'closed': +- pagure_issue_status = 'Open' +- else: +- pagure_issue_status = 'Fixed' +- +- pagure_issue_created_at = github_issue.created_at +- +- #Not sure how to deal with this atm +- pagure_issue_assignee = None +- +- if github_issue.labels: +- pagure_issue_tags = [i.name for i in github_issue.labels] +- else: +- pagure_issue_tags = [] +- +- +- #few things not supported by github +- pagure_issue_depends = [] +- pagure_issue_blocks = [] +- pagure_issue_is_private = False +- +- +- #User who created the issue +- pagure_issue_user = models.User( +- name=github_issue.user.login, +- fullname=github_issue.user.name, +- emails=[github_issue.user.email]) +- +- +- pagure_issue = models.Issue( +- id=None, +- title = pagure_issue_title, +- content = pagure_issue_content, +- status = pagure_issue_status, +- date_created = pagure_issue_created_at, +- user = pagure_issue_user.to_json(), +- private = pagure_issue_is_private, +- tags = pagure_issue_tags, +- depends = pagure_issue_depends, +- blocks = pagure_issue_blocks, +- assignee = pagure_issue_assignee) +- +- +- #comments on the issue +- comments = [] +- for comment in github_issue.get_comments(): +- +- comment_user = comment.user +- pagure_issue_comment_user_email = comment_user.email +- pagure_issue_comment_body = comment.body +- pagure_issue_comment_created_at = comment.created_at +- pagure_issue_comment_updated_at = comment.updated_at +- +- +- #No idea what to do with this right now +- #editor: not supported by github api +- pagure_issue_comment_parent = None +- pagure_issue_comment_editor = None +- +- #comment updated at +- pagure_issue_comment_edited_on = comment.updated_at +- +- #The User who commented +- pagure_issue_comment_user = models.User( +- name=comment_user.login, +- fullname=comment_user.name, +- emails=[comment_user.email] if comment_user.email \ +- else [github_get_commentor_email(comment_user.login)]) +- +- #Object to represent comment on an issue +- pagure_issue_comment = models.IssueComment( +- id=None, +- comment=pagure_issue_comment_body, +- parent=pagure_issue_comment_parent, +- date_created=pagure_issue_comment_created_at, +- user=pagure_issue_comment_user.to_json(), +- edited_on=pagure_issue_comment_edited_on, +- editor=pagure_issue_comment_editor) +- +- comments.append(pagure_issue_comment.to_json()) +- +- #add all the comments to the issue object +- pagure_issue.comments = comments +- +- #update the local git repo +- pagure_importer.lib.git.update_git(pagure_issue, repo_path, repo_folder) +diff --git a/pagure_importer/lib/sources/importer_trac.py b/pagure_importer/lib/sources/importer_trac.py +deleted file mode 100644 +index f69a3ce..0000000 +--- a/pagure_importer/lib/sources/importer_trac.py ++++ /dev/null +@@ -1,132 +0,0 @@ +-from xmlrpclib import ServerProxy +-import pagure_importer +-import pagure_importer.lib +-from pagure_importer.lib.fas import FASclient +-from pagure_importer.lib import trac +- +- +-class TracImporter(): +- '''Pagure importer for trac instance''' +- +- def __init__(self, trac_project_url): +- self.tracclient = ServerProxy(trac_project_url + '/rpc') +- self.fasclient = FASclient('user', 'password', +- 'https://admin.fedoraproject.org/accounts') +- +- def _find_fas_user(self, user): +- person = self.fasclient.person_by_username(user) +- human_name = person['human_name'] +- email = person['email'] +- pagure_user = models.User( +- name=user, +- fullname=human_name, +- emails=[email]) +- return pagure_user +- +- def _get_ticket_tags(self, trac_ticket): +- return [] +- +- def _get_ticket_status(self, trac_ticket): +- ''' Converts Trac ticket status +- to Pagure issue status''' +- +- if trac_ticket['status'] != 'closed': +- ticket_status = 'Open' +- else: +- ticket_status = 'Fixed' +- return ticket_status +- +- def _populate_comments(self, trac_comments): +- comments = [] +- for comment in trac_comments: +- if comment[2] == 'comment' and comment[4] != '': +- comment_user = comment[1] +- pagure_issue_comment_user_email = None +- pagure_issue_comment_body = comment[4] +- pagure_issue_comment_created_at = datetime.strptime( +- comment[0].value, "%Y%m%dT%H:%M:%S") +- pagure_issue_comment_updated_at = None +- +- # No idea what to do with this right now +- # editor: not supported by github api +- pagure_issue_comment_parent = None +- pagure_issue_comment_editor = None +- +- # comment updated at +- pagure_issue_comment_edited_on = None +- +- # The User who commented +- pagure_issue_comment_user = self._find_fas_user(comment[1]) +- +- # Object to represent comment on an issue +- pagure_issue_comment = models.IssueComment( +- id=None, +- comment=pagure_issue_comment_body, +- parent=pagure_issue_comment_parent, +- date_created=pagure_issue_comment_created_at, +- user=pagure_issue_comment_user.to_json(), +- edited_on=pagure_issue_comment_edited_on, +- editor=pagure_issue_comment_editor) +- +- comments.append(pagure_issue_comment.to_json()) +- return comments +- +- def _populate_issue(self, ticket_id): +- trac_ticket = self.trac.ticket.get(ticket_id)[3] +- pagure_issue_title = trac_ticket['summary'] +- pagure_issue_content = trac_ticket['description'] +- +- if pagure_issue_content == '': +- pagure_issue_content = '#No Description Provided' +- +- pagure_issue_status = self._get_ticket_status(trac_ticket) +- +- pagure_issue_created_at = datetime.strptime( +- self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") +- +- pagure_issue_assignee = trac_ticket['owner'] +- +- pagure_issue_tags = self._get_ticket_tags(trac_ticket) +- +- pagure_issue_depends = [] +- pagure_issue_blocks = [] +- pagure_issue_is_private = False +- +- pagure_issue_user = self._find_fas_user(trac_ticket['reporter']) +- +- pagure_issue = models.Issue( +- id=None, +- title=pagure_issue_title, +- content=pagure_issue_content, +- status=pagure_issue_status, +- date_created=pagure_issue_created_at, +- user=pagure_issue_user.to_json(), +- private=pagure_issue_is_private, +- tags=pagure_issue_tags, +- depends=pagure_issue_depends, +- blocks=pagure_issue_blocks, +- assignee=pagure_issue_assignee) +- return pagure_issue +- +- def import_issues(self, repo_path, repo_folder, +- trac_query='max=0&order=id'): +- '''Import issues from trac instance using xmlrpc API''' +- tickets_id = self.tracclient.ticket.query(trac_query) +- +- for ticket_id in tickets_id: +- +- pagure_issue = trac.populate_issue(self.tracclient, +- self.fasclient, ticket_id) +- +- pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) +- comments = trac.populate_comments(self.fasclient, +- pagure_issue_comments) +- +- # add all the comments to the issue object +- pagure_issue.comments = comments +- +- # update the local git repo +- print 'Update repo with issue :' + str(ticket_id) +- pagure_importer.lib.git.update_git(pagure_issue, +- repo_path, +- repo_folder) +diff --git a/pagure_importer/lib/trac.py b/pagure_importer/lib/trac.py +deleted file mode 100644 +index fb3d062..0000000 +--- a/pagure_importer/lib/trac.py ++++ /dev/null +@@ -1,90 +0,0 @@ +-from pagure_importer.lib.models import IssueComment, Issue +-from datetime import datetime +- +- +-def get_ticket_tags(trac_ticket): +- return [] +- +- +-def get_ticket_status(trac_ticket): +- ''' Converts Trac ticket status +- to Pagure issue status''' +- +- if trac_ticket['status'] != 'closed': +- ticket_status = 'Open' +- else: +- ticket_status = 'Fixed' +- return ticket_status +- +- +-def populate_comments(fasclient, trac_comments): +- comments = [] +- for comment in trac_comments: +- if comment[2] == 'comment' and comment[4] != '': +- comment_user = comment[1] +- pagure_issue_comment_user_email = None +- pagure_issue_comment_body = comment[4] +- pagure_issue_comment_created_at = datetime.strptime( +- comment[0].value, "%Y%m%dT%H:%M:%S") +- pagure_issue_comment_updated_at = None +- +- # No idea what to do with this right now +- # editor: not supported by github api +- pagure_issue_comment_parent = None +- pagure_issue_comment_editor = None +- +- # comment updated at +- pagure_issue_comment_edited_on = None +- +- # The User who commented +- pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) +- +- # Object to represent comment on an issue +- pagure_issue_comment = IssueComment( +- id=None, +- comment=pagure_issue_comment_body, +- parent=pagure_issue_comment_parent, +- date_created=pagure_issue_comment_created_at, +- user=pagure_issue_comment_user.to_json(), +- edited_on=pagure_issue_comment_edited_on, +- editor=pagure_issue_comment_editor) +- +- comments.append(pagure_issue_comment.to_json()) +- return comments +- +- +-def populate_issue(trac, fasclient, ticket_id): +- trac_ticket = trac.ticket.get(ticket_id)[3] +- pagure_issue_title = trac_ticket['summary'] +- pagure_issue_content = trac_ticket['description'] +- +- if pagure_issue_content == '': +- pagure_issue_content = '#No Description Provided' +- +- pagure_issue_status = get_ticket_status(trac_ticket) +- +- pagure_issue_created_at = datetime.strptime( +- trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") +- +- pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) +- +- pagure_issue_tags = get_ticket_tags(trac_ticket) +- +- pagure_issue_depends = [] +- pagure_issue_blocks = [] +- pagure_issue_is_private = False +- +- pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) +- pagure_issue = Issue( +- id=ticket_id, +- title=pagure_issue_title, +- content=pagure_issue_content, +- status=pagure_issue_status, +- date_created=pagure_issue_created_at, +- user=pagure_issue_user.to_json(), +- private=pagure_issue_is_private, +- tags=pagure_issue_tags, +- depends=pagure_issue_depends, +- blocks=pagure_issue_blocks, +- assignee=pagure_issue_assignee.to_json()) +- return pagure_issue +diff --git a/pagure_importer/run.py b/pagure_importer/run.py +deleted file mode 100644 +index d327228..0000000 +--- a/pagure_importer/run.py ++++ /dev/null +@@ -1,63 +0,0 @@ +-#!/usr/bin/env python +-import getpass +-from forms import form_github_issues +-from settings import IMPORT_SOURCES, IMPORT_OPTIONS, REPO_NAME, REPO_PATH +-import pagure_importer +-import pagure_importer.lib +-import pagure_importer.lib.sources +-from pagure_importer.lib.sources.importer_github import GithubImporter +-from pagure_importer.lib.sources.importer_trac import TracImporter +-from pagure_importer.lib import generate_json_for_github_contributors, \ +- generate_json_for_github_issue_commentors, \ +- assemble_github_contributors_commentors +- +- +-def github_handler(item): +- if item.lower() == 'issues': +- github_username, github_password, github_project_name = form_github_issues() +- gen_json = raw_input('Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') +- if gen_json == 'n': +- github_importer = GithubImporter( +- github_username=github_username, +- github_password=github_password, +- github_project_name=github_project_name) +- github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) +- else: +- generate_json_for_github_contributors( +- github_username, +- github_password, +- github_project_name) +- generate_json_for_github_issue_commentors( +- github_username, +- github_password, +- github_project_name) +- assemble_github_contributors_commentors() +- return +- +- +-def trac_handler(item, fedora=False): +- if item.lower() == 'issues': +- trac_url = raw_input('Enter the trac project url: ') +- trac_importer = TracImporter(trac_url) +- trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) +- +- +-def main(): +- source = raw_input('Enter source from where you want to import: ') +- if source.lower() not in IMPORT_SOURCES: +- print 'Source location not supported' +- return +- +- item = raw_input('Enter the item to be imported: ') +- if item.lower() not in IMPORT_OPTIONS[source]: +- print 'Item import not supported' +- return +- +- if source.lower() == 'github': +- github_handler(item) +- elif source.lower() == 'fedorahosted': +- trac_handler(item, fedora=True) +- return +- +-if __name__ == '__main__': +- main() +diff --git a/pagure_importer/settings.py b/pagure_importer/settings.py +deleted file mode 100644 +index 077ea40..0000000 +--- a/pagure_importer/settings.py ++++ /dev/null +@@ -1,7 +0,0 @@ +-import os +- +-IMPORT_SOURCES = ['github', 'fedorahosted'] +-IMPORT_OPTIONS = {'github': ['issues'], 'fedorahosted': ['issues']} +- +-REPO_NAME = os.environ.get('REPO_NAME', None) #this has to be a bare repo +-REPO_PATH = os.environ.get('REPO_PATH', None) #the parent of the git directory +diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py +new file mode 100644 +index 0000000..05ccdb8 +--- /dev/null ++++ b/pagure_importer/utils/__init__.py +@@ -0,0 +1,168 @@ ++import git ++import models ++ ++import csv ++import os ++import getpass ++import requests ++import json ++from github import Github ++from requests.auth import HTTPBasicAuth ++ ++from pagure_importer.utils.exceptions import FileNotFound, EmailNotFound ++ ++def generate_json_for_github_contributors(github_username, github_password, \ ++ github_project_name): ++ ''' Creates a file containing a list of dicts containing the username and ++ emails of the contributors in the given github project ++ ''' ++ ++ github_obj = Github(github_username, github_password) ++ project = github_obj.get_repo(github_project_name) ++ commits_url = project.commits_url.replace('{/sha}', '') ++ ++ page = 0 ++ contributors = [] ++ while True: ++ page += 1 ++ payload = {'page': page } ++ data_ = json.loads(requests.get(commits_url, params=payload, ++ auth=HTTPBasicAuth(github_username, github_password)).text) ++ ++ if not data_: ++ break ++ ++ for data in data_: ++ try: ++ contributor = data['commit']['committer'] ++ contributor_email = contributor['email'] ++ contributor_fullname = contributor['name'] ++ contributor_name = data['committer']['login'] ++ except TypeError: ++ print 'Maybe one of the contributors is dropped because of lack of details' ++ continue ++ ++ json_data = { ++ 'name': contributor_name, ++ 'fullname': contributor_fullname, ++ 'emails': [contributor_email] ++ } ++ ++ present = False ++ for i in contributors: ++ if i == json_data: ++ present = True ++ break ++ ++ if not present: ++ print 'contributor added: ', contributor_name ++ contributors.append(json_data) ++ ++ with open('contributors.json', 'w') as f: ++ f.write(json.dumps(contributors)) ++ ++ return ++ ++ ++def generate_json_for_github_issue_commentors(github_username, github_password, \ ++ github_project_name): ++ ''' Will create a json file containing details of all the user ++ who have commented on any issue in the given project ++ ''' ++ ++ github_obj = Github(github_username, github_password) ++ project = github_obj.get_repo(github_project_name) ++ issue_comment_url = project.issue_comment_url.replace('{/number}', '') ++ ++ page = 0 ++ issue_commentors = [] ++ while True: ++ page += 1 ++ payload = {'page': page } ++ data_ = json.loads(requests.get(issue_comment_url, params=payload, ++ auth=HTTPBasicAuth(github_username, github_password)).text) ++ ++ if not data_: ++ break ++ ++ for data in data_: ++ try: ++ commentor = data['user']['login'] ++ except TypeError: ++ print 'Maybe one of the issue commentors have been dropped because of lack of details' ++ continue ++ ++ present = False ++ for i in issue_commentors: ++ if i == commentor: ++ present = True ++ break ++ ++ if not present: ++ print 'commentor added: ', commentor ++ issue_commentors.append(commentor) ++ ++ with open('issue_commentors.json', 'w') as f: ++ f.write(json.dumps(issue_commentors)) ++ return ++ ++ ++def assemble_github_contributors_commentors(): ++ ''' It uses the files: issue_commentors.json and contributors.json ++ Assembles and creates a file: assembled_commentors.csv ++ To use: just fill the empty blocks under emails column''' ++ ++ with open('issue_commentors.json', 'r') as ic: ++ issue_names = json.load(ic) ++ ++ with open('contributors.json', 'r') as c: ++ contributors = json.load(c) ++ ++ names = [] ++ for i in issue_names: ++ found = False ++ for j in contributors: ++ if j.get('name', None) == i: ++ j['emails'] = j.get('emails')[0] ++ names.append(j) ++ found = True ++ ++ if not found: ++ d = {'name': i, 'fullname': None, 'emails': None} ++ names.append(d) ++ ++ with open('assembled_commentors.csv', 'w') as ac: ++ field_names = ['name', 'fullname', 'emails'] ++ writer = csv.DictWriter(ac, fieldnames=field_names) ++ ++ writer.writeheader() ++ for name in names: ++ writer.writerow(name) ++ ++ ++def github_get_commentor_email(name): ++ ''' Will return the issue commentor email as given in the ++ assembled_commentors.csv file ++ ''' ++ ++ if not os.path.exists('assembled_commentors.csv'): ++ raise FileNotFound('The assembled_commentors.json file must be present \ ++ Rerun the program and choose to generate the json files') ++ ++ data = [] ++ with open('assembled_commentors.csv') as ac: ++ reader = csv.DictReader(ac) ++ for row in reader: ++ data.append(dict( \ ++ (('name', row['name']), \ ++ ('fullname', row['fullname']), \ ++ ('emails', row['emails'])))) ++ ++ ++ for i in data: ++ if i.get('name', None) == name: ++ if i['emails']: ++ return str(i['emails']) ++ else: ++ raise EmailNotFound('You need to fill out all the emails of the \ ++ issue commentors') +diff --git a/pagure_importer/utils/exceptions.py b/pagure_importer/utils/exceptions.py +new file mode 100644 +index 0000000..a5ab5fe +--- /dev/null ++++ b/pagure_importer/utils/exceptions.py +@@ -0,0 +1,22 @@ ++class GithubBadCredentials(Exception): ++ ''' Raised when username/password for github is wrong ''' ++ def __init__(self, msg): ++ self.msg = msg ++ ++ ++class GithubRepoNotFound(Exception): ++ ''' Raised when the repo is not found for the user ''' ++ def __init__(self, msg): ++ self.msg = msg ++ ++ ++class FileNotFound(Exception): ++ ''' Raised when a certain file is not found ''' ++ def __init__(self, msg): ++ self.msg = msg ++ ++ ++class EmailNotFound(Exception): ++ ''' Raised when email is not found ''' ++ def __init__(self, msg): ++ self.msg = msg +diff --git a/pagure_importer/utils/fas.py b/pagure_importer/utils/fas.py +new file mode 100644 +index 0000000..f62b5f5 +--- /dev/null ++++ b/pagure_importer/utils/fas.py +@@ -0,0 +1,23 @@ ++from fedora.client.fas2 import AccountSystem ++from pagure_importer.utils.models import User ++ ++ ++class FASclient (): ++ def __init__(self, fas_username, fas_password, fas_url): ++ self.fasclient = AccountSystem(fas_url, username=fas_username, ++ password=fas_password) ++ ++ anonymous = User(name='', fullname='', emails=[]) ++ self.fasuser = {'': anonymous} ++ ++ def find_fas_user(self, user): ++ ++ if user not in self.fasuser.keys(): ++ person = self.fasclient.person_by_username(user) ++ if not person: ++ return self.fasuser[''] ++ ++ self.fasuser[user] = User(name=user, ++ fullname=person['human_name'], ++ emails=[person['email']]) ++ return self.fasuser[user] +diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py +new file mode 100644 +index 0000000..33eba48 +--- /dev/null ++++ b/pagure_importer/utils/git.py +@@ -0,0 +1,98 @@ ++''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py ++ by pingou@pingoured.fr ++''' ++ ++import shutil ++import os ++import pygit2 ++import tempfile ++import json ++ ++from repo import * ++ ++def update_git(obj, repo_path, repo_folder): ++ """ Update the given issue in its git. ++ This method forks the provided repo, add/edit the issue whose file name ++ is defined by the uid field of the issue and if there are additions/ ++ changes commit them and push them back to the original repo. ++ """ ++ ++ if not repo_folder: ++ return ++ ++ # Get the fork ++ repopath = os.path.join(repo_folder, repo_path) ++ ++ # Clone the repo into a temp folder ++ newpath = tempfile.mkdtemp(prefix='pagure-') ++ new_repo = pygit2.clone_repository(repopath, newpath) ++ ++ file_path = os.path.join(newpath, obj.uid) ++ ++ # Get the current index ++ index = new_repo.index ++ ++ # Are we adding files ++ added = False ++ if not os.path.exists(file_path): ++ added = True ++ ++ # Write down what changed ++ with open(file_path, 'w') as stream: ++ stream.write(json.dumps( ++ obj.to_json(), sort_keys=True, indent=4, ++ separators=(',', ': '))) ++ ++ # Retrieve the list of files that changed ++ diff = new_repo.diff() ++ files = [] ++ for p in diff: ++ if hasattr(p, 'new_file_path'): ++ files.append(p.new_file_path) ++ elif hasattr(p, 'delta'): ++ files.append(p.delta.new_file.path) ++ ++ # Add the changes to the index ++ if added: ++ index.add(obj.uid) ++ for filename in files: ++ index.add(filename) ++ ++ # If not change, return ++ if not files and not added: ++ shutil.rmtree(newpath) ++ return ++ ++ # See if there is a parent to this commit ++ parent = None ++ try: ++ parent = new_repo.head.get_object().oid ++ except pygit2.GitError: ++ pass ++ ++ parents = [] ++ if parent: ++ parents.append(parent) ++ ++ # Author/commiter will always be this one ++ author = pygit2.Signature(name='pagure', email='pagure') ++ ++ # Actually commit ++ new_repo.create_commit( ++ 'refs/heads/master', ++ author, ++ author, ++ 'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title), ++ new_repo.index.write_tree(), ++ parents) ++ index.write() ++ ++ # Push to origin ++ ori_remote = new_repo.remotes[0] ++ master_ref = new_repo.lookup_reference('HEAD').resolve() ++ refname = '%s:%s' % (master_ref.name, master_ref.name) ++ ++ PagureRepo.push(ori_remote, refname) ++ ++ # Remove the clone ++ shutil.rmtree(newpath) +diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py +new file mode 100644 +index 0000000..cc61370 +--- /dev/null ++++ b/pagure_importer/utils/importer_github.py +@@ -0,0 +1,138 @@ ++from github import Github ++ ++from pagure_importer.utils import models ++from pagure_importer.utils import github_get_commentor_email ++from pagure_importer.utils.git import update_git ++from pagure_importer.utils.exceptions import ( ++ GithubBadCredentials, ++ GithubRepoNotFound ++) ++ ++ ++class GithubImporter(): ++ ''' Imports from Github using PyGithub and libpagure ''' ++ def __init__( ++ self, ++ github_username, ++ github_password, ++ github_project_name): ++ self.github_username = github_username ++ self.github_password = github_password ++ self.github_project_name = github_project_name ++ self.github = Github(github_username, github_password) ++ ++ def import_issues(self, repo_path, repo_folder, status='all'): ++ ''' Imports the issues on github for ++ the given project ++ ''' ++ github_user = None ++ try: ++ github_user = self.github.get_user(self.github_username) ++ except: ++ raise GithubBadCredentials( ++ 'Given github credentials are not correct') ++ repo = self.github.get_repo(self.github_project_name) ++ try: ++ repo_name = repo.name ++ except: ++ raise GithubRepoNotFound( ++ 'Repo not found, project name wrong') ++ ++ for github_issue in repo.get_issues(state=status): ++ ++ # title of the issue ++ pagure_issue_title = github_issue.title ++ ++ # body of the issue ++ if github_issue.body: ++ pagure_issue_content = github_issue.body ++ else: ++ pagure_issue_content = '#No Description Provided' ++ ++ # Some details of a issue ++ if github_issue.state != 'closed': ++ pagure_issue_status = 'Open' ++ else: ++ pagure_issue_status = 'Fixed' ++ ++ pagure_issue_created_at = github_issue.created_at ++ ++ # Not sure how to deal with this atm ++ pagure_issue_assignee = None ++ ++ if github_issue.labels: ++ pagure_issue_tags = [i.name for i in github_issue.labels] ++ else: ++ pagure_issue_tags = [] ++ ++ ++ # few things not supported by github ++ pagure_issue_depends = [] ++ pagure_issue_blocks = [] ++ pagure_issue_is_private = False ++ ++ ++ # User who created the issue ++ pagure_issue_user = models.User( ++ name=github_issue.user.login, ++ fullname=github_issue.user.name, ++ emails=[github_issue.user.email]) ++ ++ ++ pagure_issue = models.Issue( ++ id=None, ++ title = pagure_issue_title, ++ content = pagure_issue_content, ++ status = pagure_issue_status, ++ date_created = pagure_issue_created_at, ++ user = pagure_issue_user.to_json(), ++ private = pagure_issue_is_private, ++ tags = pagure_issue_tags, ++ depends = pagure_issue_depends, ++ blocks = pagure_issue_blocks, ++ assignee = pagure_issue_assignee) ++ ++ ++ # comments on the issue ++ comments = [] ++ for comment in github_issue.get_comments(): ++ ++ comment_user = comment.user ++ pagure_issue_comment_user_email = comment_user.email ++ pagure_issue_comment_body = comment.body ++ pagure_issue_comment_created_at = comment.created_at ++ pagure_issue_comment_updated_at = comment.updated_at ++ ++ ++ # No idea what to do with this right now ++ # editor: not supported by github api ++ pagure_issue_comment_parent = None ++ pagure_issue_comment_editor = None ++ ++ # comment updated at ++ pagure_issue_comment_edited_on = comment.updated_at ++ ++ # The User who commented ++ pagure_issue_comment_user = models.User( ++ name=comment_user.login, ++ fullname=comment_user.name, ++ emails=[comment_user.email] if comment_user.email \ ++ else [github_get_commentor_email(comment_user.login)]) ++ ++ # Object to represent comment on an issue ++ pagure_issue_comment = models.IssueComment( ++ id=None, ++ comment=pagure_issue_comment_body, ++ parent=pagure_issue_comment_parent, ++ date_created=pagure_issue_comment_created_at, ++ user=pagure_issue_comment_user.to_json(), ++ edited_on=pagure_issue_comment_edited_on, ++ editor=pagure_issue_comment_editor) ++ ++ comments.append(pagure_issue_comment.to_json()) ++ ++ # add all the comments to the issue object ++ pagure_issue.comments = comments ++ ++ # update the local git repo ++ update_git(pagure_issue, repo_path, repo_folder) +diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py +new file mode 100644 +index 0000000..50f20b0 +--- /dev/null ++++ b/pagure_importer/utils/importer_trac.py +@@ -0,0 +1,35 @@ ++from xmlrpclib import ServerProxy ++from pagure_importer.utils.git import update_git ++from pagure_importer.utils import trac ++ ++ ++class TracImporter(): ++ '''Pagure importer for trac instance''' ++ ++ def __init__(self, trac_project_url, fasclient=None): ++ self.tracclient = ServerProxy(trac_project_url + '/rpc') ++ if fasclient: ++ self.fasclient = fasclient ++ ++ def import_issues(self, repo_path, repo_folder, ++ trac_query='max=0&order=id'): ++ '''Import issues from trac instance using xmlrpc API''' ++ tickets_id = self.tracclient.ticket.query(trac_query) ++ ++ for ticket_id in tickets_id: ++ ++ pagure_issue = trac.populate_issue(self.tracclient, ++ self.fasclient, ticket_id) ++ ++ pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) ++ comments = trac.populate_comments(self.fasclient, ++ pagure_issue_comments) ++ ++ # add all the comments to the issue object ++ pagure_issue.comments = comments ++ ++ # update the local git repo ++ print 'Update repo with issue :' + str(ticket_id) ++ update_git(pagure_issue, ++ repo_path, ++ repo_folder) +diff --git a/pagure_importer/utils/models.py b/pagure_importer/utils/models.py +new file mode 100644 +index 0000000..3aa3d83 +--- /dev/null ++++ b/pagure_importer/utils/models.py +@@ -0,0 +1,105 @@ ++# -*- coding: utf-8 -*- ++ ++import datetime ++import json ++import uuid ++ ++class Issue(): ++ ''' Represents an Issue ''' ++ ++ def __init__( ++ self, id, title, content, ++ status, date_created, user, private, tags, ++ depends, blocks, assignee, comments=None): ++ ++ self.id = id ++ self.title = title ++ self.content = content ++ self.status = status ++ self.date_created = date_created ++ self.user = user ++ self.private = private ++ self.tags = tags ++ self.depends = depends ++ self.blocks = blocks ++ self.assignee = assignee ++ self.comments = comments ++ self.uid = uuid.uuid4().hex ++ ++ def to_json(self): ++ ''' Returns a dictionary representation of the issue. ++ ++ ''' ++ output = { ++ 'id': self.id, ++ 'title': self.title, ++ 'content': self.content, ++ 'status': self.status, ++ 'date_created': self.date_created.strftime('%s'), ++ 'user': self.user, ++ 'private': self.private, ++ 'tags': self.tags, ++ 'depends': self.depends, ++ 'blocks': self.blocks, ++ 'assignee': self.assignee, ++ 'comments': self.comments ++ } ++ ++ return output ++ ++ @property ++ def isa(self): ++ return 'issue' ++ ++ ++class IssueComment(): ++ ''' Represent a comment for an issue ''' ++ ++ def __init__( ++ self, id, comment, date_created, ++ user, parent=None, edited_on=None, editor=None): ++ ++ self.id = id ++ self.comment = comment ++ self.parent = parent ++ self.date_created = date_created ++ self.user = user ++ self.edited_on = edited_on ++ self.editor = editor ++ ++ def to_json(self): ++ ''' Returns a dictionary representation of the issue. ''' ++ ++ output = { ++ 'id': self.id, ++ 'comment': self.comment, ++ 'parent': self.parent, ++ 'date_created': self.date_created.strftime('%s'), ++ 'user': self.user, ++ 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, ++ 'editor': self.editor or None ++ } ++ ++ return output ++ ++ ++class User(): ++ ''' Represents a User ''' ++ ++ def __init__( ++ self, name, emails, ++ fullname=None): ++ self.name = name ++ self.fullname = fullname ++ self.emails = emails ++ ++ def to_json(self): ++ ''' Return a representation of the User in a dictionary. ''' ++ ++ output = { ++ 'name': self.name, ++ 'fullname': self.fullname, ++ 'emails': self.emails ++ } ++ ++ return output +diff --git a/pagure_importer/utils/repo.py b/pagure_importer/utils/repo.py +new file mode 100644 +index 0000000..f46b31a +--- /dev/null ++++ b/pagure_importer/utils/repo.py +@@ -0,0 +1,70 @@ ++# -*- coding: utf-8 -*- ++ ++''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py ++ by pingou@pingoured.fr ++''' ++ ++ ++import pygit2 ++import sys ++ ++ ++def get_pygit2_version(): ++ ''' Return pygit2 version as a tuple of integers. ++ This is needed for correct version comparison. ++ ''' ++ return tuple([int(i) for i in pygit2.__version__.split('.')]) ++ ++ ++class PagureRepo(pygit2.Repository): ++ """ An utility class allowing to go around pygit2's inability to be ++ stable. ++ ++ """ ++ ++ @staticmethod ++ def push(remote, refname): ++ """ Push the given reference to the specified remote. """ ++ pygit2_version = get_pygit2_version() ++ if pygit2_version >= (0, 22): ++ remote.push([refname]) ++ else: ++ remote.push(refname) ++ ++ def pull(self, remote_name='origin', branch='master', force=False): ++ ''' pull changes for the specified remote (defaults to origin). ++ ++ Code from MichaelBoselowitz at: ++ https://github.com/MichaelBoselowitz/pygit2-examples/blob/ ++ 68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58 ++ licensed under the MIT license. ++ ''' ++ ++ for remote in self.remotes: ++ if remote.name == remote_name: ++ remote.fetch() ++ remote_master_id = self.lookup_reference( ++ 'refs/remotes/origin/%s' % branch).target ++ ++ if force: ++ repo_branch = self.lookup_reference( ++ 'refs/heads/%s' % branch) ++ repo_branch.set_target(remote_master_id) ++ ++ merge_result, _ = self.merge_analysis(remote_master_id) ++ # Up to date, do nothing ++ if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: ++ return ++ # We can just fastforward ++ elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: ++ self.checkout_tree(self.get(remote_master_id)) ++ master_ref = self.lookup_reference( ++ 'refs/heads/%s' % branch) ++ master_ref.set_target(remote_master_id) ++ self.head.set_target(remote_master_id) ++ elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: ++ sys.exit('Pulling remote changes leads to a conflict') ++ else: ++ print 'Unexpected merge result: %s' % ( ++ pygit2.GIT_MERGE_ANALYSIS_NORMAL) ++ raise AssertionError('Unknown merge analysis result') +diff --git a/pagure_importer/utils/trac.py b/pagure_importer/utils/trac.py +new file mode 100644 +index 0000000..d56a123 +--- /dev/null ++++ b/pagure_importer/utils/trac.py +@@ -0,0 +1,90 @@ ++from pagure_importer.utils.models import IssueComment, Issue ++from datetime import datetime ++ ++ ++def get_ticket_tags(trac_ticket): ++ return [] ++ ++ ++def get_ticket_status(trac_ticket): ++ ''' Converts Trac ticket status ++ to Pagure issue status''' ++ ++ if trac_ticket['status'] != 'closed': ++ ticket_status = 'Open' ++ else: ++ ticket_status = 'Fixed' ++ return ticket_status ++ ++ ++def populate_comments(fasclient, trac_comments): ++ comments = [] ++ for comment in trac_comments: ++ if comment[2] == 'comment' and comment[4] != '': ++ comment_user = comment[1] ++ pagure_issue_comment_user_email = None ++ pagure_issue_comment_body = comment[4] ++ pagure_issue_comment_created_at = datetime.strptime( ++ comment[0].value, "%Y%m%dT%H:%M:%S") ++ pagure_issue_comment_updated_at = None ++ ++ # No idea what to do with this right now ++ # editor: not supported by github api ++ pagure_issue_comment_parent = None ++ pagure_issue_comment_editor = None ++ ++ # comment updated at ++ pagure_issue_comment_edited_on = None ++ ++ # The User who commented ++ pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) ++ ++ # Object to represent comment on an issue ++ pagure_issue_comment = IssueComment( ++ id=None, ++ comment=pagure_issue_comment_body, ++ parent=pagure_issue_comment_parent, ++ date_created=pagure_issue_comment_created_at, ++ user=pagure_issue_comment_user.to_json(), ++ edited_on=pagure_issue_comment_edited_on, ++ editor=pagure_issue_comment_editor) ++ ++ comments.append(pagure_issue_comment.to_json()) ++ return comments ++ ++ ++def populate_issue(trac, fasclient, ticket_id): ++ trac_ticket = trac.ticket.get(ticket_id)[3] ++ pagure_issue_title = trac_ticket['summary'] ++ pagure_issue_content = trac_ticket['description'] ++ ++ if pagure_issue_content == '': ++ pagure_issue_content = '#No Description Provided' ++ ++ pagure_issue_status = get_ticket_status(trac_ticket) ++ ++ pagure_issue_created_at = datetime.strptime( ++ trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") ++ ++ pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) ++ ++ pagure_issue_tags = get_ticket_tags(trac_ticket) ++ ++ pagure_issue_depends = [] ++ pagure_issue_blocks = [] ++ pagure_issue_is_private = False ++ ++ pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) ++ pagure_issue = Issue( ++ id=ticket_id, ++ title=pagure_issue_title, ++ content=pagure_issue_content, ++ status=pagure_issue_status, ++ date_created=pagure_issue_created_at, ++ user=pagure_issue_user.to_json(), ++ private=pagure_issue_is_private, ++ tags=pagure_issue_tags, ++ depends=pagure_issue_depends, ++ blocks=pagure_issue_blocks, ++ assignee=pagure_issue_assignee.to_json()) ++ return pagure_issue +diff --git a/setup.py b/setup.py +index e031405..65fa0d5 100644 +--- a/setup.py ++++ b/setup.py +@@ -32,7 +32,7 @@ setup( + license='GNU General Public License v2.0', + entry_points={ + 'console_scripts': [ +- 'pgimport = pagure_importer.run:main' ++ 'pgimport = pagure_importer.app:app' + ], + }, + include_package_data=True, diff --git a/pagure_importer/__init__.py b/pagure_importer/__init__.py index fbab797..8b13789 100644 --- a/pagure_importer/__init__.py +++ b/pagure_importer/__init__.py @@ -1,2 +1 @@ -import lib -import settings + diff --git a/pagure_importer/app.py b/pagure_importer/app.py new file mode 100644 index 0000000..6dc3277 --- /dev/null +++ b/pagure_importer/app.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +import click +import os + +REPO_NAME = os.environ.get('REPO_NAME', None) # this has to be a bare repo +REPO_PATH = os.environ.get('REPO_PATH', None) # the parent of the git directory + + +@click.group() +def app(): + pass + +__all__ = [ + 'app', +] + +# from .commands import github +from .commands import fedorahosted +from .commands import github + +if __name__ == '__main__': + app() diff --git a/pagure_importer/commands/__init__.py b/pagure_importer/commands/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pagure_importer/commands/__init__.py diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py new file mode 100644 index 0000000..9ca8a53 --- /dev/null +++ b/pagure_importer/commands/fedorahosted.py @@ -0,0 +1,15 @@ +import click +import getpass +from pagure_importer.app import app, REPO_NAME, REPO_PATH +from pagure_importer.utils import importer_trac +from pagure_importer.utils.fas import FASclient + +@app.command() +@click.argument('project_url') +def fedorahosted(project_url): + fas_username = raw_input('Enter you FAS Username: ') + fas_password = getpass.getpass('Enter your FAS password: ') + fasclient = FASclient(fas_username, fas_password, + 'https://admin.fedoraproject.org/accounts') + trac_importer = importer_trac.TracImporter(project_url, fasclient) + trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py new file mode 100644 index 0000000..26ae4fa --- /dev/null +++ b/pagure_importer/commands/github.py @@ -0,0 +1,40 @@ +import click +import getpass + +from pagure_importer.app import app, REPO_NAME, REPO_PATH +from pagure_importer.utils.importer_github import GithubImporter +from pagure_importer.utils import ( + generate_json_for_github_contributors, + generate_json_for_github_issue_commentors, + assemble_github_contributors_commentors +) + + +def form_github_issues(): + github_username = raw_input('Enter you Github Username: ') + github_password = getpass.getpass('Enter your github password: ') + github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') + return (github_username, github_password, github_project_name) + +@app.command() +def github(): + github_username, github_password, github_project_name = form_github_issues() + gen_json = raw_input( + 'Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') + if gen_json == 'n': + github_importer = GithubImporter( + github_username=github_username, + github_password=github_password, + github_project_name=github_project_name) + github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) + else: + generate_json_for_github_contributors( + github_username, + github_password, + github_project_name) + generate_json_for_github_issue_commentors( + github_username, + github_password, + github_project_name) + assemble_github_contributors_commentors() + return diff --git a/pagure_importer/forms.py b/pagure_importer/forms.py deleted file mode 100644 index a111114..0000000 --- a/pagure_importer/forms.py +++ /dev/null @@ -1,10 +0,0 @@ -import getpass - -from lib.sources.importer_github import GithubImporter -from settings import REPO_PATH, REPO_NAME - -def form_github_issues(): - github_username = raw_input('Enter you Github Username: ') - github_password = getpass.getpass('Enter your github password: ') - github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') - return (github_username, github_password, github_project_name) diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py deleted file mode 100644 index 9a64e3b..0000000 --- a/pagure_importer/lib/__init__.py +++ /dev/null @@ -1,170 +0,0 @@ -import git -import models - -import csv -import os -import getpass -import requests -import json -from github import Github -from requests.auth import HTTPBasicAuth -import pagure_importer -import pagure_importer.lib -from pagure_importer.lib.exceptions import FileNotFound, EmailNotFound - -def generate_json_for_github_contributors(github_username, github_password, \ - github_project_name): - ''' Creates a file containing a list of dicts containing the username and - emails of the contributors in the given github project - ''' - - github_obj = Github(github_username, github_password) - project = github_obj.get_repo(github_project_name) - commits_url = project.commits_url.replace('{/sha}', '') - - page = 0 - contributors = [] - while True: - page += 1 - payload = {'page': page } - data_ = json.loads(requests.get(commits_url, params=payload, - auth=HTTPBasicAuth(github_username, github_password)).text) - - if not data_: - break - - for data in data_: - try: - contributor = data['commit']['committer'] - contributor_email = contributor['email'] - contributor_fullname = contributor['name'] - contributor_name = data['committer']['login'] - except TypeError: - print 'Maybe one of the contributors is dropped because of lack of details' - continue - - json_data = { - 'name': contributor_name, - 'fullname': contributor_fullname, - 'emails': [contributor_email] - } - - present = False - for i in contributors: - if i == json_data: - present = True - break - - if not present: - print 'contributor added: ', contributor_name - contributors.append(json_data) - - with open('contributors.json', 'w') as f: - f.write(json.dumps(contributors)) - - return - - -def generate_json_for_github_issue_commentors(github_username, github_password, \ - github_project_name): - ''' Will create a json file containing details of all the user - who have commented on any issue in the given project - ''' - - github_obj = Github(github_username, github_password) - project = github_obj.get_repo(github_project_name) - issue_comment_url = project.issue_comment_url.replace('{/number}', '') - - page = 0 - issue_commentors = [] - while True: - page += 1 - payload = {'page': page } - data_ = json.loads(requests.get(issue_comment_url, params=payload, - auth=HTTPBasicAuth(github_username, github_password)).text) - - if not data_: - break - - for data in data_: - try: - commentor = data['user']['login'] - except TypeError: - print 'Maybe one of the issue commentors have been dropped because of lack of details' - continue - - present = False - for i in issue_commentors: - if i == commentor: - present = True - break - - if not present: - print 'commentor added: ', commentor - issue_commentors.append(commentor) - - with open('issue_commentors.json', 'w') as f: - f.write(json.dumps(issue_commentors)) - return - - -def assemble_github_contributors_commentors(): - ''' It uses the files: issue_commentors.json and contributors.json - Assembles and creates a file: assembled_commentors.csv - To use: just fill the empty blocks under emails column''' - - with open('issue_commentors.json', 'r') as ic: - issue_names = json.load(ic) - - with open('contributors.json', 'r') as c: - contributors = json.load(c) - - names = [] - for i in issue_names: - found = False - for j in contributors: - if j.get('name', None) == i: - j['emails'] = j.get('emails')[0] - names.append(j) - found = True - - if not found: - d = {'name': i, 'fullname': None, 'emails': None} - names.append(d) - - with open('assembled_commentors.csv', 'w') as ac: - field_names = ['name', 'fullname', 'emails'] - writer = csv.DictWriter(ac, fieldnames=field_names) - - writer.writeheader() - for name in names: - writer.writerow(name) - - -def github_get_commentor_email(name): - ''' Will return the issue commentor email as given in the - assembled_commentors.csv file - ''' - - if not os.path.exists('assembled_commentors.csv'): - raise FileNotFound('The assembled_commentors.json file must be present \ - Rerun the program and choose to generate the json files') - - data = [] - with open('assembled_commentors.csv') as ac: - reader = csv.DictReader(ac) - for row in reader: - data.append(dict( \ - (('name', row['name']), \ - ('fullname', row['fullname']), \ - ('emails', row['emails'])))) - - - for i in data: - if i.get('name', None) == name: - if i['emails']: - return str(i['emails']) - else: - raise EmailNotFound('You need to fill out all the emails of the \ - issue commentors') - diff --git a/pagure_importer/lib/exceptions.py b/pagure_importer/lib/exceptions.py deleted file mode 100644 index a5ab5fe..0000000 --- a/pagure_importer/lib/exceptions.py +++ /dev/null @@ -1,22 +0,0 @@ -class GithubBadCredentials(Exception): - ''' Raised when username/password for github is wrong ''' - def __init__(self, msg): - self.msg = msg - - -class GithubRepoNotFound(Exception): - ''' Raised when the repo is not found for the user ''' - def __init__(self, msg): - self.msg = msg - - -class FileNotFound(Exception): - ''' Raised when a certain file is not found ''' - def __init__(self, msg): - self.msg = msg - - -class EmailNotFound(Exception): - ''' Raised when email is not found ''' - def __init__(self, msg): - self.msg = msg diff --git a/pagure_importer/lib/fas.py b/pagure_importer/lib/fas.py deleted file mode 100644 index b63f665..0000000 --- a/pagure_importer/lib/fas.py +++ /dev/null @@ -1,23 +0,0 @@ -from fedora.client.fas2 import AccountSystem -from pagure_importer.lib.models import User - - -class FASclient (): - def __init__(self, fas_username, fas_password, fas_url): - self.fasclient = AccountSystem(fas_url, username=fas_username, - password=fas_password) - - anonymous = User(name='', fullname='', emails=[]) - self.fasuser = {'': anonymous} - - def find_fas_user(self, user): - - if user not in self.fasuser.keys(): - person = self.fasclient.person_by_username(user) - if not person: - return self.fasuser[''] - - self.fasuser[user] = User(name=user, - fullname=person['human_name'], - emails=[person['email']]) - return self.fasuser[user] diff --git a/pagure_importer/lib/git.py b/pagure_importer/lib/git.py deleted file mode 100644 index 33eba48..0000000 --- a/pagure_importer/lib/git.py +++ /dev/null @@ -1,98 +0,0 @@ -''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py - by pingou@pingoured.fr -''' - -import shutil -import os -import pygit2 -import tempfile -import json - -from repo import * - -def update_git(obj, repo_path, repo_folder): - """ Update the given issue in its git. - This method forks the provided repo, add/edit the issue whose file name - is defined by the uid field of the issue and if there are additions/ - changes commit them and push them back to the original repo. - """ - - if not repo_folder: - return - - # Get the fork - repopath = os.path.join(repo_folder, repo_path) - - # Clone the repo into a temp folder - newpath = tempfile.mkdtemp(prefix='pagure-') - new_repo = pygit2.clone_repository(repopath, newpath) - - file_path = os.path.join(newpath, obj.uid) - - # Get the current index - index = new_repo.index - - # Are we adding files - added = False - if not os.path.exists(file_path): - added = True - - # Write down what changed - with open(file_path, 'w') as stream: - stream.write(json.dumps( - obj.to_json(), sort_keys=True, indent=4, - separators=(',', ': '))) - - # Retrieve the list of files that changed - diff = new_repo.diff() - files = [] - for p in diff: - if hasattr(p, 'new_file_path'): - files.append(p.new_file_path) - elif hasattr(p, 'delta'): - files.append(p.delta.new_file.path) - - # Add the changes to the index - if added: - index.add(obj.uid) - for filename in files: - index.add(filename) - - # If not change, return - if not files and not added: - shutil.rmtree(newpath) - return - - # See if there is a parent to this commit - parent = None - try: - parent = new_repo.head.get_object().oid - except pygit2.GitError: - pass - - parents = [] - if parent: - parents.append(parent) - - # Author/commiter will always be this one - author = pygit2.Signature(name='pagure', email='pagure') - - # Actually commit - new_repo.create_commit( - 'refs/heads/master', - author, - author, - 'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title), - new_repo.index.write_tree(), - parents) - index.write() - - # Push to origin - ori_remote = new_repo.remotes[0] - master_ref = new_repo.lookup_reference('HEAD').resolve() - refname = '%s:%s' % (master_ref.name, master_ref.name) - - PagureRepo.push(ori_remote, refname) - - # Remove the clone - shutil.rmtree(newpath) diff --git a/pagure_importer/lib/models.py b/pagure_importer/lib/models.py deleted file mode 100644 index 3aa3d83..0000000 --- a/pagure_importer/lib/models.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding: utf-8 -*- - -import datetime -import json -import uuid - -class Issue(): - ''' Represents an Issue ''' - - def __init__( - self, id, title, content, - status, date_created, user, private, tags, - depends, blocks, assignee, comments=None): - - self.id = id - self.title = title - self.content = content - self.status = status - self.date_created = date_created - self.user = user - self.private = private - self.tags = tags - self.depends = depends - self.blocks = blocks - self.assignee = assignee - self.comments = comments - self.uid = uuid.uuid4().hex - - def to_json(self): - ''' Returns a dictionary representation of the issue. - - ''' - output = { - 'id': self.id, - 'title': self.title, - 'content': self.content, - 'status': self.status, - 'date_created': self.date_created.strftime('%s'), - 'user': self.user, - 'private': self.private, - 'tags': self.tags, - 'depends': self.depends, - 'blocks': self.blocks, - 'assignee': self.assignee, - 'comments': self.comments - } - - return output - - @property - def isa(self): - return 'issue' - - -class IssueComment(): - ''' Represent a comment for an issue ''' - - def __init__( - self, id, comment, date_created, - user, parent=None, edited_on=None, editor=None): - - self.id = id - self.comment = comment - self.parent = parent - self.date_created = date_created - self.user = user - self.edited_on = edited_on - self.editor = editor - - def to_json(self): - ''' Returns a dictionary representation of the issue. ''' - - output = { - 'id': self.id, - 'comment': self.comment, - 'parent': self.parent, - 'date_created': self.date_created.strftime('%s'), - 'user': self.user, - 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, - 'editor': self.editor or None - } - - return output - - -class User(): - ''' Represents a User ''' - - def __init__( - self, name, emails, - fullname=None): - self.name = name - self.fullname = fullname - self.emails = emails - - def to_json(self): - ''' Return a representation of the User in a dictionary. ''' - - output = { - 'name': self.name, - 'fullname': self.fullname, - 'emails': self.emails - } - - return output diff --git a/pagure_importer/lib/repo.py b/pagure_importer/lib/repo.py deleted file mode 100644 index f46b31a..0000000 --- a/pagure_importer/lib/repo.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- - -''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py - by pingou@pingoured.fr -''' - - -import pygit2 -import sys - - -def get_pygit2_version(): - ''' Return pygit2 version as a tuple of integers. - This is needed for correct version comparison. - ''' - return tuple([int(i) for i in pygit2.__version__.split('.')]) - - -class PagureRepo(pygit2.Repository): - """ An utility class allowing to go around pygit2's inability to be - stable. - - """ - - @staticmethod - def push(remote, refname): - """ Push the given reference to the specified remote. """ - pygit2_version = get_pygit2_version() - if pygit2_version >= (0, 22): - remote.push([refname]) - else: - remote.push(refname) - - def pull(self, remote_name='origin', branch='master', force=False): - ''' pull changes for the specified remote (defaults to origin). - - Code from MichaelBoselowitz at: - https://github.com/MichaelBoselowitz/pygit2-examples/blob/ - 68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58 - licensed under the MIT license. - ''' - - for remote in self.remotes: - if remote.name == remote_name: - remote.fetch() - remote_master_id = self.lookup_reference( - 'refs/remotes/origin/%s' % branch).target - - if force: - repo_branch = self.lookup_reference( - 'refs/heads/%s' % branch) - repo_branch.set_target(remote_master_id) - - merge_result, _ = self.merge_analysis(remote_master_id) - # Up to date, do nothing - if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: - return - # We can just fastforward - elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: - self.checkout_tree(self.get(remote_master_id)) - master_ref = self.lookup_reference( - 'refs/heads/%s' % branch) - master_ref.set_target(remote_master_id) - self.head.set_target(remote_master_id) - elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: - sys.exit('Pulling remote changes leads to a conflict') - else: - print 'Unexpected merge result: %s' % ( - pygit2.GIT_MERGE_ANALYSIS_NORMAL) - raise AssertionError('Unknown merge analysis result') diff --git a/pagure_importer/lib/sources/__init__.py b/pagure_importer/lib/sources/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/pagure_importer/lib/sources/__init__.py +++ /dev/null diff --git a/pagure_importer/lib/sources/importer_github.py b/pagure_importer/lib/sources/importer_github.py deleted file mode 100644 index 2859e99..0000000 --- a/pagure_importer/lib/sources/importer_github.py +++ /dev/null @@ -1,134 +0,0 @@ -from github import Github -import pagure_importer -import pagure_importer.lib -from pagure_importer.lib import models -from pagure_importer.lib import github_get_commentor_email -from pagure_importer.lib.exceptions import GithubBadCredentials, GithubRepoNotFound - -class GithubImporter(): - ''' Imports from Github using PyGithub and libpagure ''' - def __init__( - self, - github_username, - github_password, - github_project_name): - self.github_username = github_username - self.github_password = github_password - self.github_project_name = github_project_name - self.github = Github(github_username, github_password) - - def import_issues(self, repo_path, repo_folder, status='all'): - ''' Imports the issues on github for - the given project - ''' - github_user = None - try: - github_user = self.github.get_user(self.github_username) - except: - raise GithubBadCredentials( - 'Given github credentials are not correct') - repo = self.github.get_repo(self.github_project_name) - try: - repo_name = repo.name - except: - raise GithubRepoNotFound( - 'Repo not found, project name wrong') - - for github_issue in repo.get_issues(state=status): - - #title of the issue - pagure_issue_title = github_issue.title - - #body of the issue - if github_issue.body: - pagure_issue_content = github_issue.body - else: - pagure_issue_content = '#No Description Provided' - - #Some details of a issue - if github_issue.state != 'closed': - pagure_issue_status = 'Open' - else: - pagure_issue_status = 'Fixed' - - pagure_issue_created_at = github_issue.created_at - - #Not sure how to deal with this atm - pagure_issue_assignee = None - - if github_issue.labels: - pagure_issue_tags = [i.name for i in github_issue.labels] - else: - pagure_issue_tags = [] - - - #few things not supported by github - pagure_issue_depends = [] - pagure_issue_blocks = [] - pagure_issue_is_private = False - - - #User who created the issue - pagure_issue_user = models.User( - name=github_issue.user.login, - fullname=github_issue.user.name, - emails=[github_issue.user.email]) - - - pagure_issue = models.Issue( - id=None, - title = pagure_issue_title, - content = pagure_issue_content, - status = pagure_issue_status, - date_created = pagure_issue_created_at, - user = pagure_issue_user.to_json(), - private = pagure_issue_is_private, - tags = pagure_issue_tags, - depends = pagure_issue_depends, - blocks = pagure_issue_blocks, - assignee = pagure_issue_assignee) - - - #comments on the issue - comments = [] - for comment in github_issue.get_comments(): - - comment_user = comment.user - pagure_issue_comment_user_email = comment_user.email - pagure_issue_comment_body = comment.body - pagure_issue_comment_created_at = comment.created_at - pagure_issue_comment_updated_at = comment.updated_at - - - #No idea what to do with this right now - #editor: not supported by github api - pagure_issue_comment_parent = None - pagure_issue_comment_editor = None - - #comment updated at - pagure_issue_comment_edited_on = comment.updated_at - - #The User who commented - pagure_issue_comment_user = models.User( - name=comment_user.login, - fullname=comment_user.name, - emails=[comment_user.email] if comment_user.email \ - else [github_get_commentor_email(comment_user.login)]) - - #Object to represent comment on an issue - pagure_issue_comment = models.IssueComment( - id=None, - comment=pagure_issue_comment_body, - parent=pagure_issue_comment_parent, - date_created=pagure_issue_comment_created_at, - user=pagure_issue_comment_user.to_json(), - edited_on=pagure_issue_comment_edited_on, - editor=pagure_issue_comment_editor) - - comments.append(pagure_issue_comment.to_json()) - - #add all the comments to the issue object - pagure_issue.comments = comments - - #update the local git repo - pagure_importer.lib.git.update_git(pagure_issue, repo_path, repo_folder) diff --git a/pagure_importer/lib/sources/importer_trac.py b/pagure_importer/lib/sources/importer_trac.py deleted file mode 100644 index f69a3ce..0000000 --- a/pagure_importer/lib/sources/importer_trac.py +++ /dev/null @@ -1,132 +0,0 @@ -from xmlrpclib import ServerProxy -import pagure_importer -import pagure_importer.lib -from pagure_importer.lib.fas import FASclient -from pagure_importer.lib import trac - - -class TracImporter(): - '''Pagure importer for trac instance''' - - def __init__(self, trac_project_url): - self.tracclient = ServerProxy(trac_project_url + '/rpc') - self.fasclient = FASclient('user', 'password', - 'https://admin.fedoraproject.org/accounts') - - def _find_fas_user(self, user): - person = self.fasclient.person_by_username(user) - human_name = person['human_name'] - email = person['email'] - pagure_user = models.User( - name=user, - fullname=human_name, - emails=[email]) - return pagure_user - - def _get_ticket_tags(self, trac_ticket): - return [] - - def _get_ticket_status(self, trac_ticket): - ''' Converts Trac ticket status - to Pagure issue status''' - - if trac_ticket['status'] != 'closed': - ticket_status = 'Open' - else: - ticket_status = 'Fixed' - return ticket_status - - def _populate_comments(self, trac_comments): - comments = [] - for comment in trac_comments: - if comment[2] == 'comment' and comment[4] != '': - comment_user = comment[1] - pagure_issue_comment_user_email = None - pagure_issue_comment_body = comment[4] - pagure_issue_comment_created_at = datetime.strptime( - comment[0].value, "%Y%m%dT%H:%M:%S") - pagure_issue_comment_updated_at = None - - # No idea what to do with this right now - # editor: not supported by github api - pagure_issue_comment_parent = None - pagure_issue_comment_editor = None - - # comment updated at - pagure_issue_comment_edited_on = None - - # The User who commented - pagure_issue_comment_user = self._find_fas_user(comment[1]) - - # Object to represent comment on an issue - pagure_issue_comment = models.IssueComment( - id=None, - comment=pagure_issue_comment_body, - parent=pagure_issue_comment_parent, - date_created=pagure_issue_comment_created_at, - user=pagure_issue_comment_user.to_json(), - edited_on=pagure_issue_comment_edited_on, - editor=pagure_issue_comment_editor) - - comments.append(pagure_issue_comment.to_json()) - return comments - - def _populate_issue(self, ticket_id): - trac_ticket = self.trac.ticket.get(ticket_id)[3] - pagure_issue_title = trac_ticket['summary'] - pagure_issue_content = trac_ticket['description'] - - if pagure_issue_content == '': - pagure_issue_content = '#No Description Provided' - - pagure_issue_status = self._get_ticket_status(trac_ticket) - - pagure_issue_created_at = datetime.strptime( - self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") - - pagure_issue_assignee = trac_ticket['owner'] - - pagure_issue_tags = self._get_ticket_tags(trac_ticket) - - pagure_issue_depends = [] - pagure_issue_blocks = [] - pagure_issue_is_private = False - - pagure_issue_user = self._find_fas_user(trac_ticket['reporter']) - - pagure_issue = models.Issue( - id=None, - title=pagure_issue_title, - content=pagure_issue_content, - status=pagure_issue_status, - date_created=pagure_issue_created_at, - user=pagure_issue_user.to_json(), - private=pagure_issue_is_private, - tags=pagure_issue_tags, - depends=pagure_issue_depends, - blocks=pagure_issue_blocks, - assignee=pagure_issue_assignee) - return pagure_issue - - def import_issues(self, repo_path, repo_folder, - trac_query='max=0&order=id'): - '''Import issues from trac instance using xmlrpc API''' - tickets_id = self.tracclient.ticket.query(trac_query) - - for ticket_id in tickets_id: - - pagure_issue = trac.populate_issue(self.tracclient, - self.fasclient, ticket_id) - - pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) - comments = trac.populate_comments(self.fasclient, - pagure_issue_comments) - - # add all the comments to the issue object - pagure_issue.comments = comments - - # update the local git repo - print 'Update repo with issue :' + str(ticket_id) - pagure_importer.lib.git.update_git(pagure_issue, - repo_path, - repo_folder) diff --git a/pagure_importer/lib/trac.py b/pagure_importer/lib/trac.py deleted file mode 100644 index fb3d062..0000000 --- a/pagure_importer/lib/trac.py +++ /dev/null @@ -1,90 +0,0 @@ -from pagure_importer.lib.models import IssueComment, Issue -from datetime import datetime - - -def get_ticket_tags(trac_ticket): - return [] - - -def get_ticket_status(trac_ticket): - ''' Converts Trac ticket status - to Pagure issue status''' - - if trac_ticket['status'] != 'closed': - ticket_status = 'Open' - else: - ticket_status = 'Fixed' - return ticket_status - - -def populate_comments(fasclient, trac_comments): - comments = [] - for comment in trac_comments: - if comment[2] == 'comment' and comment[4] != '': - comment_user = comment[1] - pagure_issue_comment_user_email = None - pagure_issue_comment_body = comment[4] - pagure_issue_comment_created_at = datetime.strptime( - comment[0].value, "%Y%m%dT%H:%M:%S") - pagure_issue_comment_updated_at = None - - # No idea what to do with this right now - # editor: not supported by github api - pagure_issue_comment_parent = None - pagure_issue_comment_editor = None - - # comment updated at - pagure_issue_comment_edited_on = None - - # The User who commented - pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) - - # Object to represent comment on an issue - pagure_issue_comment = IssueComment( - id=None, - comment=pagure_issue_comment_body, - parent=pagure_issue_comment_parent, - date_created=pagure_issue_comment_created_at, - user=pagure_issue_comment_user.to_json(), - edited_on=pagure_issue_comment_edited_on, - editor=pagure_issue_comment_editor) - - comments.append(pagure_issue_comment.to_json()) - return comments - - -def populate_issue(trac, fasclient, ticket_id): - trac_ticket = trac.ticket.get(ticket_id)[3] - pagure_issue_title = trac_ticket['summary'] - pagure_issue_content = trac_ticket['description'] - - if pagure_issue_content == '': - pagure_issue_content = '#No Description Provided' - - pagure_issue_status = get_ticket_status(trac_ticket) - - pagure_issue_created_at = datetime.strptime( - trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") - - pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) - - pagure_issue_tags = get_ticket_tags(trac_ticket) - - pagure_issue_depends = [] - pagure_issue_blocks = [] - pagure_issue_is_private = False - - pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) - pagure_issue = Issue( - id=ticket_id, - title=pagure_issue_title, - content=pagure_issue_content, - status=pagure_issue_status, - date_created=pagure_issue_created_at, - user=pagure_issue_user.to_json(), - private=pagure_issue_is_private, - tags=pagure_issue_tags, - depends=pagure_issue_depends, - blocks=pagure_issue_blocks, - assignee=pagure_issue_assignee.to_json()) - return pagure_issue diff --git a/pagure_importer/run.py b/pagure_importer/run.py deleted file mode 100644 index d327228..0000000 --- a/pagure_importer/run.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -import getpass -from forms import form_github_issues -from settings import IMPORT_SOURCES, IMPORT_OPTIONS, REPO_NAME, REPO_PATH -import pagure_importer -import pagure_importer.lib -import pagure_importer.lib.sources -from pagure_importer.lib.sources.importer_github import GithubImporter -from pagure_importer.lib.sources.importer_trac import TracImporter -from pagure_importer.lib import generate_json_for_github_contributors, \ - generate_json_for_github_issue_commentors, \ - assemble_github_contributors_commentors - - -def github_handler(item): - if item.lower() == 'issues': - github_username, github_password, github_project_name = form_github_issues() - gen_json = raw_input('Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') - if gen_json == 'n': - github_importer = GithubImporter( - github_username=github_username, - github_password=github_password, - github_project_name=github_project_name) - github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) - else: - generate_json_for_github_contributors( - github_username, - github_password, - github_project_name) - generate_json_for_github_issue_commentors( - github_username, - github_password, - github_project_name) - assemble_github_contributors_commentors() - return - - -def trac_handler(item, fedora=False): - if item.lower() == 'issues': - trac_url = raw_input('Enter the trac project url: ') - trac_importer = TracImporter(trac_url) - trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) - - -def main(): - source = raw_input('Enter source from where you want to import: ') - if source.lower() not in IMPORT_SOURCES: - print 'Source location not supported' - return - - item = raw_input('Enter the item to be imported: ') - if item.lower() not in IMPORT_OPTIONS[source]: - print 'Item import not supported' - return - - if source.lower() == 'github': - github_handler(item) - elif source.lower() == 'fedorahosted': - trac_handler(item, fedora=True) - return - -if __name__ == '__main__': - main() diff --git a/pagure_importer/settings.py b/pagure_importer/settings.py deleted file mode 100644 index 077ea40..0000000 --- a/pagure_importer/settings.py +++ /dev/null @@ -1,7 +0,0 @@ -import os - -IMPORT_SOURCES = ['github', 'fedorahosted'] -IMPORT_OPTIONS = {'github': ['issues'], 'fedorahosted': ['issues']} - -REPO_NAME = os.environ.get('REPO_NAME', None) #this has to be a bare repo -REPO_PATH = os.environ.get('REPO_PATH', None) #the parent of the git directory diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py new file mode 100644 index 0000000..05ccdb8 --- /dev/null +++ b/pagure_importer/utils/__init__.py @@ -0,0 +1,168 @@ +import git +import models + +import csv +import os +import getpass +import requests +import json +from github import Github +from requests.auth import HTTPBasicAuth + +from pagure_importer.utils.exceptions import FileNotFound, EmailNotFound + +def generate_json_for_github_contributors(github_username, github_password, \ + github_project_name): + ''' Creates a file containing a list of dicts containing the username and + emails of the contributors in the given github project + ''' + + github_obj = Github(github_username, github_password) + project = github_obj.get_repo(github_project_name) + commits_url = project.commits_url.replace('{/sha}', '') + + page = 0 + contributors = [] + while True: + page += 1 + payload = {'page': page } + data_ = json.loads(requests.get(commits_url, params=payload, + auth=HTTPBasicAuth(github_username, github_password)).text) + + if not data_: + break + + for data in data_: + try: + contributor = data['commit']['committer'] + contributor_email = contributor['email'] + contributor_fullname = contributor['name'] + contributor_name = data['committer']['login'] + except TypeError: + print 'Maybe one of the contributors is dropped because of lack of details' + continue + + json_data = { + 'name': contributor_name, + 'fullname': contributor_fullname, + 'emails': [contributor_email] + } + + present = False + for i in contributors: + if i == json_data: + present = True + break + + if not present: + print 'contributor added: ', contributor_name + contributors.append(json_data) + + with open('contributors.json', 'w') as f: + f.write(json.dumps(contributors)) + + return + + +def generate_json_for_github_issue_commentors(github_username, github_password, \ + github_project_name): + ''' Will create a json file containing details of all the user + who have commented on any issue in the given project + ''' + + github_obj = Github(github_username, github_password) + project = github_obj.get_repo(github_project_name) + issue_comment_url = project.issue_comment_url.replace('{/number}', '') + + page = 0 + issue_commentors = [] + while True: + page += 1 + payload = {'page': page } + data_ = json.loads(requests.get(issue_comment_url, params=payload, + auth=HTTPBasicAuth(github_username, github_password)).text) + + if not data_: + break + + for data in data_: + try: + commentor = data['user']['login'] + except TypeError: + print 'Maybe one of the issue commentors have been dropped because of lack of details' + continue + + present = False + for i in issue_commentors: + if i == commentor: + present = True + break + + if not present: + print 'commentor added: ', commentor + issue_commentors.append(commentor) + + with open('issue_commentors.json', 'w') as f: + f.write(json.dumps(issue_commentors)) + return + + +def assemble_github_contributors_commentors(): + ''' It uses the files: issue_commentors.json and contributors.json + Assembles and creates a file: assembled_commentors.csv + To use: just fill the empty blocks under emails column''' + + with open('issue_commentors.json', 'r') as ic: + issue_names = json.load(ic) + + with open('contributors.json', 'r') as c: + contributors = json.load(c) + + names = [] + for i in issue_names: + found = False + for j in contributors: + if j.get('name', None) == i: + j['emails'] = j.get('emails')[0] + names.append(j) + found = True + + if not found: + d = {'name': i, 'fullname': None, 'emails': None} + names.append(d) + + with open('assembled_commentors.csv', 'w') as ac: + field_names = ['name', 'fullname', 'emails'] + writer = csv.DictWriter(ac, fieldnames=field_names) + + writer.writeheader() + for name in names: + writer.writerow(name) + + +def github_get_commentor_email(name): + ''' Will return the issue commentor email as given in the + assembled_commentors.csv file + ''' + + if not os.path.exists('assembled_commentors.csv'): + raise FileNotFound('The assembled_commentors.json file must be present \ + Rerun the program and choose to generate the json files') + + data = [] + with open('assembled_commentors.csv') as ac: + reader = csv.DictReader(ac) + for row in reader: + data.append(dict( \ + (('name', row['name']), \ + ('fullname', row['fullname']), \ + ('emails', row['emails'])))) + + + for i in data: + if i.get('name', None) == name: + if i['emails']: + return str(i['emails']) + else: + raise EmailNotFound('You need to fill out all the emails of the \ + issue commentors') diff --git a/pagure_importer/utils/exceptions.py b/pagure_importer/utils/exceptions.py new file mode 100644 index 0000000..a5ab5fe --- /dev/null +++ b/pagure_importer/utils/exceptions.py @@ -0,0 +1,22 @@ +class GithubBadCredentials(Exception): + ''' Raised when username/password for github is wrong ''' + def __init__(self, msg): + self.msg = msg + + +class GithubRepoNotFound(Exception): + ''' Raised when the repo is not found for the user ''' + def __init__(self, msg): + self.msg = msg + + +class FileNotFound(Exception): + ''' Raised when a certain file is not found ''' + def __init__(self, msg): + self.msg = msg + + +class EmailNotFound(Exception): + ''' Raised when email is not found ''' + def __init__(self, msg): + self.msg = msg diff --git a/pagure_importer/utils/fas.py b/pagure_importer/utils/fas.py new file mode 100644 index 0000000..f62b5f5 --- /dev/null +++ b/pagure_importer/utils/fas.py @@ -0,0 +1,23 @@ +from fedora.client.fas2 import AccountSystem +from pagure_importer.utils.models import User + + +class FASclient (): + def __init__(self, fas_username, fas_password, fas_url): + self.fasclient = AccountSystem(fas_url, username=fas_username, + password=fas_password) + + anonymous = User(name='', fullname='', emails=[]) + self.fasuser = {'': anonymous} + + def find_fas_user(self, user): + + if user not in self.fasuser.keys(): + person = self.fasclient.person_by_username(user) + if not person: + return self.fasuser[''] + + self.fasuser[user] = User(name=user, + fullname=person['human_name'], + emails=[person['email']]) + return self.fasuser[user] diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py new file mode 100644 index 0000000..33eba48 --- /dev/null +++ b/pagure_importer/utils/git.py @@ -0,0 +1,98 @@ +''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py + by pingou@pingoured.fr +''' + +import shutil +import os +import pygit2 +import tempfile +import json + +from repo import * + +def update_git(obj, repo_path, repo_folder): + """ Update the given issue in its git. + This method forks the provided repo, add/edit the issue whose file name + is defined by the uid field of the issue and if there are additions/ + changes commit them and push them back to the original repo. + """ + + if not repo_folder: + return + + # Get the fork + repopath = os.path.join(repo_folder, repo_path) + + # Clone the repo into a temp folder + newpath = tempfile.mkdtemp(prefix='pagure-') + new_repo = pygit2.clone_repository(repopath, newpath) + + file_path = os.path.join(newpath, obj.uid) + + # Get the current index + index = new_repo.index + + # Are we adding files + added = False + if not os.path.exists(file_path): + added = True + + # Write down what changed + with open(file_path, 'w') as stream: + stream.write(json.dumps( + obj.to_json(), sort_keys=True, indent=4, + separators=(',', ': '))) + + # Retrieve the list of files that changed + diff = new_repo.diff() + files = [] + for p in diff: + if hasattr(p, 'new_file_path'): + files.append(p.new_file_path) + elif hasattr(p, 'delta'): + files.append(p.delta.new_file.path) + + # Add the changes to the index + if added: + index.add(obj.uid) + for filename in files: + index.add(filename) + + # If not change, return + if not files and not added: + shutil.rmtree(newpath) + return + + # See if there is a parent to this commit + parent = None + try: + parent = new_repo.head.get_object().oid + except pygit2.GitError: + pass + + parents = [] + if parent: + parents.append(parent) + + # Author/commiter will always be this one + author = pygit2.Signature(name='pagure', email='pagure') + + # Actually commit + new_repo.create_commit( + 'refs/heads/master', + author, + author, + 'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title), + new_repo.index.write_tree(), + parents) + index.write() + + # Push to origin + ori_remote = new_repo.remotes[0] + master_ref = new_repo.lookup_reference('HEAD').resolve() + refname = '%s:%s' % (master_ref.name, master_ref.name) + + PagureRepo.push(ori_remote, refname) + + # Remove the clone + shutil.rmtree(newpath) diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py new file mode 100644 index 0000000..cc61370 --- /dev/null +++ b/pagure_importer/utils/importer_github.py @@ -0,0 +1,138 @@ +from github import Github + +from pagure_importer.utils import models +from pagure_importer.utils import github_get_commentor_email +from pagure_importer.utils.git import update_git +from pagure_importer.utils.exceptions import ( + GithubBadCredentials, + GithubRepoNotFound +) + + +class GithubImporter(): + ''' Imports from Github using PyGithub and libpagure ''' + def __init__( + self, + github_username, + github_password, + github_project_name): + self.github_username = github_username + self.github_password = github_password + self.github_project_name = github_project_name + self.github = Github(github_username, github_password) + + def import_issues(self, repo_path, repo_folder, status='all'): + ''' Imports the issues on github for + the given project + ''' + github_user = None + try: + github_user = self.github.get_user(self.github_username) + except: + raise GithubBadCredentials( + 'Given github credentials are not correct') + repo = self.github.get_repo(self.github_project_name) + try: + repo_name = repo.name + except: + raise GithubRepoNotFound( + 'Repo not found, project name wrong') + + for github_issue in repo.get_issues(state=status): + + # title of the issue + pagure_issue_title = github_issue.title + + # body of the issue + if github_issue.body: + pagure_issue_content = github_issue.body + else: + pagure_issue_content = '#No Description Provided' + + # Some details of a issue + if github_issue.state != 'closed': + pagure_issue_status = 'Open' + else: + pagure_issue_status = 'Fixed' + + pagure_issue_created_at = github_issue.created_at + + # Not sure how to deal with this atm + pagure_issue_assignee = None + + if github_issue.labels: + pagure_issue_tags = [i.name for i in github_issue.labels] + else: + pagure_issue_tags = [] + + + # few things not supported by github + pagure_issue_depends = [] + pagure_issue_blocks = [] + pagure_issue_is_private = False + + + # User who created the issue + pagure_issue_user = models.User( + name=github_issue.user.login, + fullname=github_issue.user.name, + emails=[github_issue.user.email]) + + + pagure_issue = models.Issue( + id=None, + title = pagure_issue_title, + content = pagure_issue_content, + status = pagure_issue_status, + date_created = pagure_issue_created_at, + user = pagure_issue_user.to_json(), + private = pagure_issue_is_private, + tags = pagure_issue_tags, + depends = pagure_issue_depends, + blocks = pagure_issue_blocks, + assignee = pagure_issue_assignee) + + + # comments on the issue + comments = [] + for comment in github_issue.get_comments(): + + comment_user = comment.user + pagure_issue_comment_user_email = comment_user.email + pagure_issue_comment_body = comment.body + pagure_issue_comment_created_at = comment.created_at + pagure_issue_comment_updated_at = comment.updated_at + + + # No idea what to do with this right now + # editor: not supported by github api + pagure_issue_comment_parent = None + pagure_issue_comment_editor = None + + # comment updated at + pagure_issue_comment_edited_on = comment.updated_at + + # The User who commented + pagure_issue_comment_user = models.User( + name=comment_user.login, + fullname=comment_user.name, + emails=[comment_user.email] if comment_user.email \ + else [github_get_commentor_email(comment_user.login)]) + + # Object to represent comment on an issue + pagure_issue_comment = models.IssueComment( + id=None, + comment=pagure_issue_comment_body, + parent=pagure_issue_comment_parent, + date_created=pagure_issue_comment_created_at, + user=pagure_issue_comment_user.to_json(), + edited_on=pagure_issue_comment_edited_on, + editor=pagure_issue_comment_editor) + + comments.append(pagure_issue_comment.to_json()) + + # add all the comments to the issue object + pagure_issue.comments = comments + + # update the local git repo + update_git(pagure_issue, repo_path, repo_folder) diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py new file mode 100644 index 0000000..8e7fff8 --- /dev/null +++ b/pagure_importer/utils/importer_trac.py @@ -0,0 +1,32 @@ +from xmlrpclib import ServerProxy +from pagure_importer.utils.git import update_git +from pagure_importer.utils import trac + + +class TracImporter(): + '''Pagure importer for trac instance''' + + def __init__(self, trac_project_url, fasclient=None): + self.tracclient = ServerProxy(trac_project_url + '/rpc') + self.fasclient = fasclient + + def import_issues(self, repo_path, repo_folder, + trac_query='max=0&order=id'): + '''Import issues from trac instance using xmlrpc API''' + tickets_id = self.tracclient.ticket.query(trac_query) + + for ticket_id in tickets_id: + + pagure_issue = trac.populate_issue(self.tracclient, + self.fasclient, ticket_id) + + pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) + comments = trac.populate_comments(self.fasclient, + pagure_issue_comments) + + # add all the comments to the issue object + pagure_issue.comments = comments + + # update the local git repo + print 'Update repo with issue :' + str(ticket_id) + update_git(pagure_issue, repo_path, repo_folder) diff --git a/pagure_importer/utils/models.py b/pagure_importer/utils/models.py new file mode 100644 index 0000000..3aa3d83 --- /dev/null +++ b/pagure_importer/utils/models.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- + +import datetime +import json +import uuid + +class Issue(): + ''' Represents an Issue ''' + + def __init__( + self, id, title, content, + status, date_created, user, private, tags, + depends, blocks, assignee, comments=None): + + self.id = id + self.title = title + self.content = content + self.status = status + self.date_created = date_created + self.user = user + self.private = private + self.tags = tags + self.depends = depends + self.blocks = blocks + self.assignee = assignee + self.comments = comments + self.uid = uuid.uuid4().hex + + def to_json(self): + ''' Returns a dictionary representation of the issue. + + ''' + output = { + 'id': self.id, + 'title': self.title, + 'content': self.content, + 'status': self.status, + 'date_created': self.date_created.strftime('%s'), + 'user': self.user, + 'private': self.private, + 'tags': self.tags, + 'depends': self.depends, + 'blocks': self.blocks, + 'assignee': self.assignee, + 'comments': self.comments + } + + return output + + @property + def isa(self): + return 'issue' + + +class IssueComment(): + ''' Represent a comment for an issue ''' + + def __init__( + self, id, comment, date_created, + user, parent=None, edited_on=None, editor=None): + + self.id = id + self.comment = comment + self.parent = parent + self.date_created = date_created + self.user = user + self.edited_on = edited_on + self.editor = editor + + def to_json(self): + ''' Returns a dictionary representation of the issue. ''' + + output = { + 'id': self.id, + 'comment': self.comment, + 'parent': self.parent, + 'date_created': self.date_created.strftime('%s'), + 'user': self.user, + 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, + 'editor': self.editor or None + } + + return output + + +class User(): + ''' Represents a User ''' + + def __init__( + self, name, emails, + fullname=None): + self.name = name + self.fullname = fullname + self.emails = emails + + def to_json(self): + ''' Return a representation of the User in a dictionary. ''' + + output = { + 'name': self.name, + 'fullname': self.fullname, + 'emails': self.emails + } + + return output diff --git a/pagure_importer/utils/repo.py b/pagure_importer/utils/repo.py new file mode 100644 index 0000000..f46b31a --- /dev/null +++ b/pagure_importer/utils/repo.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py + by pingou@pingoured.fr +''' + + +import pygit2 +import sys + + +def get_pygit2_version(): + ''' Return pygit2 version as a tuple of integers. + This is needed for correct version comparison. + ''' + return tuple([int(i) for i in pygit2.__version__.split('.')]) + + +class PagureRepo(pygit2.Repository): + """ An utility class allowing to go around pygit2's inability to be + stable. + + """ + + @staticmethod + def push(remote, refname): + """ Push the given reference to the specified remote. """ + pygit2_version = get_pygit2_version() + if pygit2_version >= (0, 22): + remote.push([refname]) + else: + remote.push(refname) + + def pull(self, remote_name='origin', branch='master', force=False): + ''' pull changes for the specified remote (defaults to origin). + + Code from MichaelBoselowitz at: + https://github.com/MichaelBoselowitz/pygit2-examples/blob/ + 68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58 + licensed under the MIT license. + ''' + + for remote in self.remotes: + if remote.name == remote_name: + remote.fetch() + remote_master_id = self.lookup_reference( + 'refs/remotes/origin/%s' % branch).target + + if force: + repo_branch = self.lookup_reference( + 'refs/heads/%s' % branch) + repo_branch.set_target(remote_master_id) + + merge_result, _ = self.merge_analysis(remote_master_id) + # Up to date, do nothing + if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: + return + # We can just fastforward + elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: + self.checkout_tree(self.get(remote_master_id)) + master_ref = self.lookup_reference( + 'refs/heads/%s' % branch) + master_ref.set_target(remote_master_id) + self.head.set_target(remote_master_id) + elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: + sys.exit('Pulling remote changes leads to a conflict') + else: + print 'Unexpected merge result: %s' % ( + pygit2.GIT_MERGE_ANALYSIS_NORMAL) + raise AssertionError('Unknown merge analysis result') diff --git a/pagure_importer/utils/trac.py b/pagure_importer/utils/trac.py new file mode 100644 index 0000000..d56a123 --- /dev/null +++ b/pagure_importer/utils/trac.py @@ -0,0 +1,90 @@ +from pagure_importer.utils.models import IssueComment, Issue +from datetime import datetime + + +def get_ticket_tags(trac_ticket): + return [] + + +def get_ticket_status(trac_ticket): + ''' Converts Trac ticket status + to Pagure issue status''' + + if trac_ticket['status'] != 'closed': + ticket_status = 'Open' + else: + ticket_status = 'Fixed' + return ticket_status + + +def populate_comments(fasclient, trac_comments): + comments = [] + for comment in trac_comments: + if comment[2] == 'comment' and comment[4] != '': + comment_user = comment[1] + pagure_issue_comment_user_email = None + pagure_issue_comment_body = comment[4] + pagure_issue_comment_created_at = datetime.strptime( + comment[0].value, "%Y%m%dT%H:%M:%S") + pagure_issue_comment_updated_at = None + + # No idea what to do with this right now + # editor: not supported by github api + pagure_issue_comment_parent = None + pagure_issue_comment_editor = None + + # comment updated at + pagure_issue_comment_edited_on = None + + # The User who commented + pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) + + # Object to represent comment on an issue + pagure_issue_comment = IssueComment( + id=None, + comment=pagure_issue_comment_body, + parent=pagure_issue_comment_parent, + date_created=pagure_issue_comment_created_at, + user=pagure_issue_comment_user.to_json(), + edited_on=pagure_issue_comment_edited_on, + editor=pagure_issue_comment_editor) + + comments.append(pagure_issue_comment.to_json()) + return comments + + +def populate_issue(trac, fasclient, ticket_id): + trac_ticket = trac.ticket.get(ticket_id)[3] + pagure_issue_title = trac_ticket['summary'] + pagure_issue_content = trac_ticket['description'] + + if pagure_issue_content == '': + pagure_issue_content = '#No Description Provided' + + pagure_issue_status = get_ticket_status(trac_ticket) + + pagure_issue_created_at = datetime.strptime( + trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") + + pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) + + pagure_issue_tags = get_ticket_tags(trac_ticket) + + pagure_issue_depends = [] + pagure_issue_blocks = [] + pagure_issue_is_private = False + + pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) + pagure_issue = Issue( + id=ticket_id, + title=pagure_issue_title, + content=pagure_issue_content, + status=pagure_issue_status, + date_created=pagure_issue_created_at, + user=pagure_issue_user.to_json(), + private=pagure_issue_is_private, + tags=pagure_issue_tags, + depends=pagure_issue_depends, + blocks=pagure_issue_blocks, + assignee=pagure_issue_assignee.to_json()) + return pagure_issue diff --git a/setup.py b/setup.py index e031405..65fa0d5 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( license='GNU General Public License v2.0', entry_points={ 'console_scripts': [ - 'pgimport = pagure_importer.run:main' + 'pgimport = pagure_importer.app:app' ], }, include_package_data=True, From d1338f49cdbc38864fe6f7f734dbfa0a39f50325 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 04 2016 19:22:34 +0000 Subject: [PATCH 6/24] Deal with non FAS user, for generic Trac instance --- diff --git a/pagure_importer/utils/trac.py b/pagure_importer/utils/trac.py index d56a123..4847959 100644 --- a/pagure_importer/utils/trac.py +++ b/pagure_importer/utils/trac.py @@ -1,4 +1,4 @@ -from pagure_importer.utils.models import IssueComment, Issue +from pagure_importer.utils.models import IssueComment, Issue, User from datetime import datetime @@ -37,7 +37,12 @@ def populate_comments(fasclient, trac_comments): pagure_issue_comment_edited_on = None # The User who commented - pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) + if fasclient: + pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) + else: + pagure_issue_comment_user = User(name='', + fullname='', + emails=[]) # Object to represent comment on an issue pagure_issue_comment = IssueComment( @@ -66,7 +71,13 @@ def populate_issue(trac, fasclient, ticket_id): pagure_issue_created_at = datetime.strptime( trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") - pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) + if fasclient: + pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) + pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) + else: + anonymous = User(name='', fullname='', emails=[]) + pagure_issue_assignee = anonymous + pagure_issue_user = anonymous pagure_issue_tags = get_ticket_tags(trac_ticket) @@ -74,7 +85,6 @@ def populate_issue(trac, fasclient, ticket_id): pagure_issue_blocks = [] pagure_issue_is_private = False - pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) pagure_issue = Issue( id=ticket_id, title=pagure_issue_title, From f0f28f906131a3e04c175786f815c2181f952c6f Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 04 2016 19:29:07 +0000 Subject: [PATCH 7/24] updated requirements.txt --- diff --git a/diff b/diff deleted file mode 100644 index 2a92e6e..0000000 --- a/diff +++ /dev/null @@ -1,1922 +0,0 @@ -diff --git a/pagure_importer/__init__.py b/pagure_importer/__init__.py -index fbab797..8b13789 100644 ---- a/pagure_importer/__init__.py -+++ b/pagure_importer/__init__.py -@@ -1,2 +1 @@ --import lib --import settings -+ -diff --git a/pagure_importer/app.py b/pagure_importer/app.py -new file mode 100644 -index 0000000..6dc3277 ---- /dev/null -+++ b/pagure_importer/app.py -@@ -0,0 +1,23 @@ -+#!/usr/bin/env python -+ -+import click -+import os -+ -+REPO_NAME = os.environ.get('REPO_NAME', None) # this has to be a bare repo -+REPO_PATH = os.environ.get('REPO_PATH', None) # the parent of the git directory -+ -+ -+@click.group() -+def app(): -+ pass -+ -+__all__ = [ -+ 'app', -+] -+ -+# from .commands import github -+from .commands import fedorahosted -+from .commands import github -+ -+if __name__ == '__main__': -+ app() -diff --git a/pagure_importer/commands/__init__.py b/pagure_importer/commands/__init__.py -new file mode 100644 -index 0000000..e69de29 -diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py -new file mode 100644 -index 0000000..650d133 ---- /dev/null -+++ b/pagure_importer/commands/fedorahosted.py -@@ -0,0 +1,15 @@ -+import click -+import getpass -+from pagure_importer.app import app, REPO_NAME, REPO_PATH -+from pagure_importer.utils import importer_trac -+from pagure_importer.utils.fas import FASclient -+ -+@app.command() -+@click.argument('project_url') -+def fedorahosted(project_url): -+ fas_username = raw_input('Enter you Github Username: ') -+ fas_password = getpass.getpass('Enter your github password: ') -+ fasclient = FASclient(fas_username, fas_password, -+ 'https://admin.fedoraproject.org/accounts') -+ trac_importer = importer_trac.TracImporter(project_url, fasclient) -+ trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) -diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py -new file mode 100644 -index 0000000..26ae4fa ---- /dev/null -+++ b/pagure_importer/commands/github.py -@@ -0,0 +1,40 @@ -+import click -+import getpass -+ -+from pagure_importer.app import app, REPO_NAME, REPO_PATH -+from pagure_importer.utils.importer_github import GithubImporter -+from pagure_importer.utils import ( -+ generate_json_for_github_contributors, -+ generate_json_for_github_issue_commentors, -+ assemble_github_contributors_commentors -+) -+ -+ -+def form_github_issues(): -+ github_username = raw_input('Enter you Github Username: ') -+ github_password = getpass.getpass('Enter your github password: ') -+ github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') -+ return (github_username, github_password, github_project_name) -+ -+@app.command() -+def github(): -+ github_username, github_password, github_project_name = form_github_issues() -+ gen_json = raw_input( -+ 'Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') -+ if gen_json == 'n': -+ github_importer = GithubImporter( -+ github_username=github_username, -+ github_password=github_password, -+ github_project_name=github_project_name) -+ github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) -+ else: -+ generate_json_for_github_contributors( -+ github_username, -+ github_password, -+ github_project_name) -+ generate_json_for_github_issue_commentors( -+ github_username, -+ github_password, -+ github_project_name) -+ assemble_github_contributors_commentors() -+ return -diff --git a/pagure_importer/forms.py b/pagure_importer/forms.py -deleted file mode 100644 -index a111114..0000000 ---- a/pagure_importer/forms.py -+++ /dev/null -@@ -1,10 +0,0 @@ --import getpass -- --from lib.sources.importer_github import GithubImporter --from settings import REPO_PATH, REPO_NAME -- --def form_github_issues(): -- github_username = raw_input('Enter you Github Username: ') -- github_password = getpass.getpass('Enter your github password: ') -- github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') -- return (github_username, github_password, github_project_name) -diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py -deleted file mode 100644 -index 9a64e3b..0000000 ---- a/pagure_importer/lib/__init__.py -+++ /dev/null -@@ -1,170 +0,0 @@ --import git --import models -- --import csv --import os --import getpass --import requests --import json --from github import Github --from requests.auth import HTTPBasicAuth --import pagure_importer --import pagure_importer.lib --from pagure_importer.lib.exceptions import FileNotFound, EmailNotFound -- --def generate_json_for_github_contributors(github_username, github_password, \ -- github_project_name): -- ''' Creates a file containing a list of dicts containing the username and -- emails of the contributors in the given github project -- ''' -- -- github_obj = Github(github_username, github_password) -- project = github_obj.get_repo(github_project_name) -- commits_url = project.commits_url.replace('{/sha}', '') -- -- page = 0 -- contributors = [] -- while True: -- page += 1 -- payload = {'page': page } -- data_ = json.loads(requests.get(commits_url, params=payload, -- auth=HTTPBasicAuth(github_username, github_password)).text) -- -- if not data_: -- break -- -- for data in data_: -- try: -- contributor = data['commit']['committer'] -- contributor_email = contributor['email'] -- contributor_fullname = contributor['name'] -- contributor_name = data['committer']['login'] -- except TypeError: -- print 'Maybe one of the contributors is dropped because of lack of details' -- continue -- -- json_data = { -- 'name': contributor_name, -- 'fullname': contributor_fullname, -- 'emails': [contributor_email] -- } -- -- present = False -- for i in contributors: -- if i == json_data: -- present = True -- break -- -- if not present: -- print 'contributor added: ', contributor_name -- contributors.append(json_data) -- -- with open('contributors.json', 'w') as f: -- f.write(json.dumps(contributors)) -- -- return -- -- --def generate_json_for_github_issue_commentors(github_username, github_password, \ -- github_project_name): -- ''' Will create a json file containing details of all the user -- who have commented on any issue in the given project -- ''' -- -- github_obj = Github(github_username, github_password) -- project = github_obj.get_repo(github_project_name) -- issue_comment_url = project.issue_comment_url.replace('{/number}', '') -- -- page = 0 -- issue_commentors = [] -- while True: -- page += 1 -- payload = {'page': page } -- data_ = json.loads(requests.get(issue_comment_url, params=payload, -- auth=HTTPBasicAuth(github_username, github_password)).text) -- -- if not data_: -- break -- -- for data in data_: -- try: -- commentor = data['user']['login'] -- except TypeError: -- print 'Maybe one of the issue commentors have been dropped because of lack of details' -- continue -- -- present = False -- for i in issue_commentors: -- if i == commentor: -- present = True -- break -- -- if not present: -- print 'commentor added: ', commentor -- issue_commentors.append(commentor) -- -- with open('issue_commentors.json', 'w') as f: -- f.write(json.dumps(issue_commentors)) -- return -- -- --def assemble_github_contributors_commentors(): -- ''' It uses the files: issue_commentors.json and contributors.json -- Assembles and creates a file: assembled_commentors.csv -- To use: just fill the empty blocks under emails column''' -- -- with open('issue_commentors.json', 'r') as ic: -- issue_names = json.load(ic) -- -- with open('contributors.json', 'r') as c: -- contributors = json.load(c) -- -- names = [] -- for i in issue_names: -- found = False -- for j in contributors: -- if j.get('name', None) == i: -- j['emails'] = j.get('emails')[0] -- names.append(j) -- found = True -- -- if not found: -- d = {'name': i, 'fullname': None, 'emails': None} -- names.append(d) -- -- with open('assembled_commentors.csv', 'w') as ac: -- field_names = ['name', 'fullname', 'emails'] -- writer = csv.DictWriter(ac, fieldnames=field_names) -- -- writer.writeheader() -- for name in names: -- writer.writerow(name) -- -- --def github_get_commentor_email(name): -- ''' Will return the issue commentor email as given in the -- assembled_commentors.csv file -- ''' -- -- if not os.path.exists('assembled_commentors.csv'): -- raise FileNotFound('The assembled_commentors.json file must be present \ -- Rerun the program and choose to generate the json files') -- -- data = [] -- with open('assembled_commentors.csv') as ac: -- reader = csv.DictReader(ac) -- for row in reader: -- data.append(dict( \ -- (('name', row['name']), \ -- ('fullname', row['fullname']), \ -- ('emails', row['emails'])))) -- -- -- for i in data: -- if i.get('name', None) == name: -- if i['emails']: -- return str(i['emails']) -- else: -- raise EmailNotFound('You need to fill out all the emails of the \ -- issue commentors') -- -diff --git a/pagure_importer/lib/exceptions.py b/pagure_importer/lib/exceptions.py -deleted file mode 100644 -index a5ab5fe..0000000 ---- a/pagure_importer/lib/exceptions.py -+++ /dev/null -@@ -1,22 +0,0 @@ --class GithubBadCredentials(Exception): -- ''' Raised when username/password for github is wrong ''' -- def __init__(self, msg): -- self.msg = msg -- -- --class GithubRepoNotFound(Exception): -- ''' Raised when the repo is not found for the user ''' -- def __init__(self, msg): -- self.msg = msg -- -- --class FileNotFound(Exception): -- ''' Raised when a certain file is not found ''' -- def __init__(self, msg): -- self.msg = msg -- -- --class EmailNotFound(Exception): -- ''' Raised when email is not found ''' -- def __init__(self, msg): -- self.msg = msg -diff --git a/pagure_importer/lib/fas.py b/pagure_importer/lib/fas.py -deleted file mode 100644 -index b63f665..0000000 ---- a/pagure_importer/lib/fas.py -+++ /dev/null -@@ -1,23 +0,0 @@ --from fedora.client.fas2 import AccountSystem --from pagure_importer.lib.models import User -- -- --class FASclient (): -- def __init__(self, fas_username, fas_password, fas_url): -- self.fasclient = AccountSystem(fas_url, username=fas_username, -- password=fas_password) -- -- anonymous = User(name='', fullname='', emails=[]) -- self.fasuser = {'': anonymous} -- -- def find_fas_user(self, user): -- -- if user not in self.fasuser.keys(): -- person = self.fasclient.person_by_username(user) -- if not person: -- return self.fasuser[''] -- -- self.fasuser[user] = User(name=user, -- fullname=person['human_name'], -- emails=[person['email']]) -- return self.fasuser[user] -diff --git a/pagure_importer/lib/git.py b/pagure_importer/lib/git.py -deleted file mode 100644 -index 33eba48..0000000 ---- a/pagure_importer/lib/git.py -+++ /dev/null -@@ -1,98 +0,0 @@ --''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py -- by pingou@pingoured.fr --''' -- --import shutil --import os --import pygit2 --import tempfile --import json -- --from repo import * -- --def update_git(obj, repo_path, repo_folder): -- """ Update the given issue in its git. -- This method forks the provided repo, add/edit the issue whose file name -- is defined by the uid field of the issue and if there are additions/ -- changes commit them and push them back to the original repo. -- """ -- -- if not repo_folder: -- return -- -- # Get the fork -- repopath = os.path.join(repo_folder, repo_path) -- -- # Clone the repo into a temp folder -- newpath = tempfile.mkdtemp(prefix='pagure-') -- new_repo = pygit2.clone_repository(repopath, newpath) -- -- file_path = os.path.join(newpath, obj.uid) -- -- # Get the current index -- index = new_repo.index -- -- # Are we adding files -- added = False -- if not os.path.exists(file_path): -- added = True -- -- # Write down what changed -- with open(file_path, 'w') as stream: -- stream.write(json.dumps( -- obj.to_json(), sort_keys=True, indent=4, -- separators=(',', ': '))) -- -- # Retrieve the list of files that changed -- diff = new_repo.diff() -- files = [] -- for p in diff: -- if hasattr(p, 'new_file_path'): -- files.append(p.new_file_path) -- elif hasattr(p, 'delta'): -- files.append(p.delta.new_file.path) -- -- # Add the changes to the index -- if added: -- index.add(obj.uid) -- for filename in files: -- index.add(filename) -- -- # If not change, return -- if not files and not added: -- shutil.rmtree(newpath) -- return -- -- # See if there is a parent to this commit -- parent = None -- try: -- parent = new_repo.head.get_object().oid -- except pygit2.GitError: -- pass -- -- parents = [] -- if parent: -- parents.append(parent) -- -- # Author/commiter will always be this one -- author = pygit2.Signature(name='pagure', email='pagure') -- -- # Actually commit -- new_repo.create_commit( -- 'refs/heads/master', -- author, -- author, -- 'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title), -- new_repo.index.write_tree(), -- parents) -- index.write() -- -- # Push to origin -- ori_remote = new_repo.remotes[0] -- master_ref = new_repo.lookup_reference('HEAD').resolve() -- refname = '%s:%s' % (master_ref.name, master_ref.name) -- -- PagureRepo.push(ori_remote, refname) -- -- # Remove the clone -- shutil.rmtree(newpath) -diff --git a/pagure_importer/lib/models.py b/pagure_importer/lib/models.py -deleted file mode 100644 -index 3aa3d83..0000000 ---- a/pagure_importer/lib/models.py -+++ /dev/null -@@ -1,105 +0,0 @@ --# -*- coding: utf-8 -*- -- --import datetime --import json --import uuid -- --class Issue(): -- ''' Represents an Issue ''' -- -- def __init__( -- self, id, title, content, -- status, date_created, user, private, tags, -- depends, blocks, assignee, comments=None): -- -- self.id = id -- self.title = title -- self.content = content -- self.status = status -- self.date_created = date_created -- self.user = user -- self.private = private -- self.tags = tags -- self.depends = depends -- self.blocks = blocks -- self.assignee = assignee -- self.comments = comments -- self.uid = uuid.uuid4().hex -- -- def to_json(self): -- ''' Returns a dictionary representation of the issue. -- -- ''' -- output = { -- 'id': self.id, -- 'title': self.title, -- 'content': self.content, -- 'status': self.status, -- 'date_created': self.date_created.strftime('%s'), -- 'user': self.user, -- 'private': self.private, -- 'tags': self.tags, -- 'depends': self.depends, -- 'blocks': self.blocks, -- 'assignee': self.assignee, -- 'comments': self.comments -- } -- -- return output -- -- @property -- def isa(self): -- return 'issue' -- -- --class IssueComment(): -- ''' Represent a comment for an issue ''' -- -- def __init__( -- self, id, comment, date_created, -- user, parent=None, edited_on=None, editor=None): -- -- self.id = id -- self.comment = comment -- self.parent = parent -- self.date_created = date_created -- self.user = user -- self.edited_on = edited_on -- self.editor = editor -- -- def to_json(self): -- ''' Returns a dictionary representation of the issue. ''' -- -- output = { -- 'id': self.id, -- 'comment': self.comment, -- 'parent': self.parent, -- 'date_created': self.date_created.strftime('%s'), -- 'user': self.user, -- 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, -- 'editor': self.editor or None -- } -- -- return output -- -- --class User(): -- ''' Represents a User ''' -- -- def __init__( -- self, name, emails, -- fullname=None): -- self.name = name -- self.fullname = fullname -- self.emails = emails -- -- def to_json(self): -- ''' Return a representation of the User in a dictionary. ''' -- -- output = { -- 'name': self.name, -- 'fullname': self.fullname, -- 'emails': self.emails -- } -- -- return output -diff --git a/pagure_importer/lib/repo.py b/pagure_importer/lib/repo.py -deleted file mode 100644 -index f46b31a..0000000 ---- a/pagure_importer/lib/repo.py -+++ /dev/null -@@ -1,70 +0,0 @@ --# -*- coding: utf-8 -*- -- --''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py -- by pingou@pingoured.fr --''' -- -- --import pygit2 --import sys -- -- --def get_pygit2_version(): -- ''' Return pygit2 version as a tuple of integers. -- This is needed for correct version comparison. -- ''' -- return tuple([int(i) for i in pygit2.__version__.split('.')]) -- -- --class PagureRepo(pygit2.Repository): -- """ An utility class allowing to go around pygit2's inability to be -- stable. -- -- """ -- -- @staticmethod -- def push(remote, refname): -- """ Push the given reference to the specified remote. """ -- pygit2_version = get_pygit2_version() -- if pygit2_version >= (0, 22): -- remote.push([refname]) -- else: -- remote.push(refname) -- -- def pull(self, remote_name='origin', branch='master', force=False): -- ''' pull changes for the specified remote (defaults to origin). -- -- Code from MichaelBoselowitz at: -- https://github.com/MichaelBoselowitz/pygit2-examples/blob/ -- 68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58 -- licensed under the MIT license. -- ''' -- -- for remote in self.remotes: -- if remote.name == remote_name: -- remote.fetch() -- remote_master_id = self.lookup_reference( -- 'refs/remotes/origin/%s' % branch).target -- -- if force: -- repo_branch = self.lookup_reference( -- 'refs/heads/%s' % branch) -- repo_branch.set_target(remote_master_id) -- -- merge_result, _ = self.merge_analysis(remote_master_id) -- # Up to date, do nothing -- if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: -- return -- # We can just fastforward -- elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: -- self.checkout_tree(self.get(remote_master_id)) -- master_ref = self.lookup_reference( -- 'refs/heads/%s' % branch) -- master_ref.set_target(remote_master_id) -- self.head.set_target(remote_master_id) -- elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: -- sys.exit('Pulling remote changes leads to a conflict') -- else: -- print 'Unexpected merge result: %s' % ( -- pygit2.GIT_MERGE_ANALYSIS_NORMAL) -- raise AssertionError('Unknown merge analysis result') -diff --git a/pagure_importer/lib/sources/__init__.py b/pagure_importer/lib/sources/__init__.py -deleted file mode 100644 -index e69de29..0000000 -diff --git a/pagure_importer/lib/sources/importer_github.py b/pagure_importer/lib/sources/importer_github.py -deleted file mode 100644 -index 2859e99..0000000 ---- a/pagure_importer/lib/sources/importer_github.py -+++ /dev/null -@@ -1,134 +0,0 @@ --from github import Github --import pagure_importer --import pagure_importer.lib --from pagure_importer.lib import models --from pagure_importer.lib import github_get_commentor_email --from pagure_importer.lib.exceptions import GithubBadCredentials, GithubRepoNotFound -- --class GithubImporter(): -- ''' Imports from Github using PyGithub and libpagure ''' -- def __init__( -- self, -- github_username, -- github_password, -- github_project_name): -- self.github_username = github_username -- self.github_password = github_password -- self.github_project_name = github_project_name -- self.github = Github(github_username, github_password) -- -- def import_issues(self, repo_path, repo_folder, status='all'): -- ''' Imports the issues on github for -- the given project -- ''' -- github_user = None -- try: -- github_user = self.github.get_user(self.github_username) -- except: -- raise GithubBadCredentials( -- 'Given github credentials are not correct') -- repo = self.github.get_repo(self.github_project_name) -- try: -- repo_name = repo.name -- except: -- raise GithubRepoNotFound( -- 'Repo not found, project name wrong') -- -- for github_issue in repo.get_issues(state=status): -- -- #title of the issue -- pagure_issue_title = github_issue.title -- -- #body of the issue -- if github_issue.body: -- pagure_issue_content = github_issue.body -- else: -- pagure_issue_content = '#No Description Provided' -- -- #Some details of a issue -- if github_issue.state != 'closed': -- pagure_issue_status = 'Open' -- else: -- pagure_issue_status = 'Fixed' -- -- pagure_issue_created_at = github_issue.created_at -- -- #Not sure how to deal with this atm -- pagure_issue_assignee = None -- -- if github_issue.labels: -- pagure_issue_tags = [i.name for i in github_issue.labels] -- else: -- pagure_issue_tags = [] -- -- -- #few things not supported by github -- pagure_issue_depends = [] -- pagure_issue_blocks = [] -- pagure_issue_is_private = False -- -- -- #User who created the issue -- pagure_issue_user = models.User( -- name=github_issue.user.login, -- fullname=github_issue.user.name, -- emails=[github_issue.user.email]) -- -- -- pagure_issue = models.Issue( -- id=None, -- title = pagure_issue_title, -- content = pagure_issue_content, -- status = pagure_issue_status, -- date_created = pagure_issue_created_at, -- user = pagure_issue_user.to_json(), -- private = pagure_issue_is_private, -- tags = pagure_issue_tags, -- depends = pagure_issue_depends, -- blocks = pagure_issue_blocks, -- assignee = pagure_issue_assignee) -- -- -- #comments on the issue -- comments = [] -- for comment in github_issue.get_comments(): -- -- comment_user = comment.user -- pagure_issue_comment_user_email = comment_user.email -- pagure_issue_comment_body = comment.body -- pagure_issue_comment_created_at = comment.created_at -- pagure_issue_comment_updated_at = comment.updated_at -- -- -- #No idea what to do with this right now -- #editor: not supported by github api -- pagure_issue_comment_parent = None -- pagure_issue_comment_editor = None -- -- #comment updated at -- pagure_issue_comment_edited_on = comment.updated_at -- -- #The User who commented -- pagure_issue_comment_user = models.User( -- name=comment_user.login, -- fullname=comment_user.name, -- emails=[comment_user.email] if comment_user.email \ -- else [github_get_commentor_email(comment_user.login)]) -- -- #Object to represent comment on an issue -- pagure_issue_comment = models.IssueComment( -- id=None, -- comment=pagure_issue_comment_body, -- parent=pagure_issue_comment_parent, -- date_created=pagure_issue_comment_created_at, -- user=pagure_issue_comment_user.to_json(), -- edited_on=pagure_issue_comment_edited_on, -- editor=pagure_issue_comment_editor) -- -- comments.append(pagure_issue_comment.to_json()) -- -- #add all the comments to the issue object -- pagure_issue.comments = comments -- -- #update the local git repo -- pagure_importer.lib.git.update_git(pagure_issue, repo_path, repo_folder) -diff --git a/pagure_importer/lib/sources/importer_trac.py b/pagure_importer/lib/sources/importer_trac.py -deleted file mode 100644 -index f69a3ce..0000000 ---- a/pagure_importer/lib/sources/importer_trac.py -+++ /dev/null -@@ -1,132 +0,0 @@ --from xmlrpclib import ServerProxy --import pagure_importer --import pagure_importer.lib --from pagure_importer.lib.fas import FASclient --from pagure_importer.lib import trac -- -- --class TracImporter(): -- '''Pagure importer for trac instance''' -- -- def __init__(self, trac_project_url): -- self.tracclient = ServerProxy(trac_project_url + '/rpc') -- self.fasclient = FASclient('user', 'password', -- 'https://admin.fedoraproject.org/accounts') -- -- def _find_fas_user(self, user): -- person = self.fasclient.person_by_username(user) -- human_name = person['human_name'] -- email = person['email'] -- pagure_user = models.User( -- name=user, -- fullname=human_name, -- emails=[email]) -- return pagure_user -- -- def _get_ticket_tags(self, trac_ticket): -- return [] -- -- def _get_ticket_status(self, trac_ticket): -- ''' Converts Trac ticket status -- to Pagure issue status''' -- -- if trac_ticket['status'] != 'closed': -- ticket_status = 'Open' -- else: -- ticket_status = 'Fixed' -- return ticket_status -- -- def _populate_comments(self, trac_comments): -- comments = [] -- for comment in trac_comments: -- if comment[2] == 'comment' and comment[4] != '': -- comment_user = comment[1] -- pagure_issue_comment_user_email = None -- pagure_issue_comment_body = comment[4] -- pagure_issue_comment_created_at = datetime.strptime( -- comment[0].value, "%Y%m%dT%H:%M:%S") -- pagure_issue_comment_updated_at = None -- -- # No idea what to do with this right now -- # editor: not supported by github api -- pagure_issue_comment_parent = None -- pagure_issue_comment_editor = None -- -- # comment updated at -- pagure_issue_comment_edited_on = None -- -- # The User who commented -- pagure_issue_comment_user = self._find_fas_user(comment[1]) -- -- # Object to represent comment on an issue -- pagure_issue_comment = models.IssueComment( -- id=None, -- comment=pagure_issue_comment_body, -- parent=pagure_issue_comment_parent, -- date_created=pagure_issue_comment_created_at, -- user=pagure_issue_comment_user.to_json(), -- edited_on=pagure_issue_comment_edited_on, -- editor=pagure_issue_comment_editor) -- -- comments.append(pagure_issue_comment.to_json()) -- return comments -- -- def _populate_issue(self, ticket_id): -- trac_ticket = self.trac.ticket.get(ticket_id)[3] -- pagure_issue_title = trac_ticket['summary'] -- pagure_issue_content = trac_ticket['description'] -- -- if pagure_issue_content == '': -- pagure_issue_content = '#No Description Provided' -- -- pagure_issue_status = self._get_ticket_status(trac_ticket) -- -- pagure_issue_created_at = datetime.strptime( -- self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") -- -- pagure_issue_assignee = trac_ticket['owner'] -- -- pagure_issue_tags = self._get_ticket_tags(trac_ticket) -- -- pagure_issue_depends = [] -- pagure_issue_blocks = [] -- pagure_issue_is_private = False -- -- pagure_issue_user = self._find_fas_user(trac_ticket['reporter']) -- -- pagure_issue = models.Issue( -- id=None, -- title=pagure_issue_title, -- content=pagure_issue_content, -- status=pagure_issue_status, -- date_created=pagure_issue_created_at, -- user=pagure_issue_user.to_json(), -- private=pagure_issue_is_private, -- tags=pagure_issue_tags, -- depends=pagure_issue_depends, -- blocks=pagure_issue_blocks, -- assignee=pagure_issue_assignee) -- return pagure_issue -- -- def import_issues(self, repo_path, repo_folder, -- trac_query='max=0&order=id'): -- '''Import issues from trac instance using xmlrpc API''' -- tickets_id = self.tracclient.ticket.query(trac_query) -- -- for ticket_id in tickets_id: -- -- pagure_issue = trac.populate_issue(self.tracclient, -- self.fasclient, ticket_id) -- -- pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) -- comments = trac.populate_comments(self.fasclient, -- pagure_issue_comments) -- -- # add all the comments to the issue object -- pagure_issue.comments = comments -- -- # update the local git repo -- print 'Update repo with issue :' + str(ticket_id) -- pagure_importer.lib.git.update_git(pagure_issue, -- repo_path, -- repo_folder) -diff --git a/pagure_importer/lib/trac.py b/pagure_importer/lib/trac.py -deleted file mode 100644 -index fb3d062..0000000 ---- a/pagure_importer/lib/trac.py -+++ /dev/null -@@ -1,90 +0,0 @@ --from pagure_importer.lib.models import IssueComment, Issue --from datetime import datetime -- -- --def get_ticket_tags(trac_ticket): -- return [] -- -- --def get_ticket_status(trac_ticket): -- ''' Converts Trac ticket status -- to Pagure issue status''' -- -- if trac_ticket['status'] != 'closed': -- ticket_status = 'Open' -- else: -- ticket_status = 'Fixed' -- return ticket_status -- -- --def populate_comments(fasclient, trac_comments): -- comments = [] -- for comment in trac_comments: -- if comment[2] == 'comment' and comment[4] != '': -- comment_user = comment[1] -- pagure_issue_comment_user_email = None -- pagure_issue_comment_body = comment[4] -- pagure_issue_comment_created_at = datetime.strptime( -- comment[0].value, "%Y%m%dT%H:%M:%S") -- pagure_issue_comment_updated_at = None -- -- # No idea what to do with this right now -- # editor: not supported by github api -- pagure_issue_comment_parent = None -- pagure_issue_comment_editor = None -- -- # comment updated at -- pagure_issue_comment_edited_on = None -- -- # The User who commented -- pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) -- -- # Object to represent comment on an issue -- pagure_issue_comment = IssueComment( -- id=None, -- comment=pagure_issue_comment_body, -- parent=pagure_issue_comment_parent, -- date_created=pagure_issue_comment_created_at, -- user=pagure_issue_comment_user.to_json(), -- edited_on=pagure_issue_comment_edited_on, -- editor=pagure_issue_comment_editor) -- -- comments.append(pagure_issue_comment.to_json()) -- return comments -- -- --def populate_issue(trac, fasclient, ticket_id): -- trac_ticket = trac.ticket.get(ticket_id)[3] -- pagure_issue_title = trac_ticket['summary'] -- pagure_issue_content = trac_ticket['description'] -- -- if pagure_issue_content == '': -- pagure_issue_content = '#No Description Provided' -- -- pagure_issue_status = get_ticket_status(trac_ticket) -- -- pagure_issue_created_at = datetime.strptime( -- trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") -- -- pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) -- -- pagure_issue_tags = get_ticket_tags(trac_ticket) -- -- pagure_issue_depends = [] -- pagure_issue_blocks = [] -- pagure_issue_is_private = False -- -- pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) -- pagure_issue = Issue( -- id=ticket_id, -- title=pagure_issue_title, -- content=pagure_issue_content, -- status=pagure_issue_status, -- date_created=pagure_issue_created_at, -- user=pagure_issue_user.to_json(), -- private=pagure_issue_is_private, -- tags=pagure_issue_tags, -- depends=pagure_issue_depends, -- blocks=pagure_issue_blocks, -- assignee=pagure_issue_assignee.to_json()) -- return pagure_issue -diff --git a/pagure_importer/run.py b/pagure_importer/run.py -deleted file mode 100644 -index d327228..0000000 ---- a/pagure_importer/run.py -+++ /dev/null -@@ -1,63 +0,0 @@ --#!/usr/bin/env python --import getpass --from forms import form_github_issues --from settings import IMPORT_SOURCES, IMPORT_OPTIONS, REPO_NAME, REPO_PATH --import pagure_importer --import pagure_importer.lib --import pagure_importer.lib.sources --from pagure_importer.lib.sources.importer_github import GithubImporter --from pagure_importer.lib.sources.importer_trac import TracImporter --from pagure_importer.lib import generate_json_for_github_contributors, \ -- generate_json_for_github_issue_commentors, \ -- assemble_github_contributors_commentors -- -- --def github_handler(item): -- if item.lower() == 'issues': -- github_username, github_password, github_project_name = form_github_issues() -- gen_json = raw_input('Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') -- if gen_json == 'n': -- github_importer = GithubImporter( -- github_username=github_username, -- github_password=github_password, -- github_project_name=github_project_name) -- github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) -- else: -- generate_json_for_github_contributors( -- github_username, -- github_password, -- github_project_name) -- generate_json_for_github_issue_commentors( -- github_username, -- github_password, -- github_project_name) -- assemble_github_contributors_commentors() -- return -- -- --def trac_handler(item, fedora=False): -- if item.lower() == 'issues': -- trac_url = raw_input('Enter the trac project url: ') -- trac_importer = TracImporter(trac_url) -- trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) -- -- --def main(): -- source = raw_input('Enter source from where you want to import: ') -- if source.lower() not in IMPORT_SOURCES: -- print 'Source location not supported' -- return -- -- item = raw_input('Enter the item to be imported: ') -- if item.lower() not in IMPORT_OPTIONS[source]: -- print 'Item import not supported' -- return -- -- if source.lower() == 'github': -- github_handler(item) -- elif source.lower() == 'fedorahosted': -- trac_handler(item, fedora=True) -- return -- --if __name__ == '__main__': -- main() -diff --git a/pagure_importer/settings.py b/pagure_importer/settings.py -deleted file mode 100644 -index 077ea40..0000000 ---- a/pagure_importer/settings.py -+++ /dev/null -@@ -1,7 +0,0 @@ --import os -- --IMPORT_SOURCES = ['github', 'fedorahosted'] --IMPORT_OPTIONS = {'github': ['issues'], 'fedorahosted': ['issues']} -- --REPO_NAME = os.environ.get('REPO_NAME', None) #this has to be a bare repo --REPO_PATH = os.environ.get('REPO_PATH', None) #the parent of the git directory -diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py -new file mode 100644 -index 0000000..05ccdb8 ---- /dev/null -+++ b/pagure_importer/utils/__init__.py -@@ -0,0 +1,168 @@ -+import git -+import models -+ -+import csv -+import os -+import getpass -+import requests -+import json -+from github import Github -+from requests.auth import HTTPBasicAuth -+ -+from pagure_importer.utils.exceptions import FileNotFound, EmailNotFound -+ -+def generate_json_for_github_contributors(github_username, github_password, \ -+ github_project_name): -+ ''' Creates a file containing a list of dicts containing the username and -+ emails of the contributors in the given github project -+ ''' -+ -+ github_obj = Github(github_username, github_password) -+ project = github_obj.get_repo(github_project_name) -+ commits_url = project.commits_url.replace('{/sha}', '') -+ -+ page = 0 -+ contributors = [] -+ while True: -+ page += 1 -+ payload = {'page': page } -+ data_ = json.loads(requests.get(commits_url, params=payload, -+ auth=HTTPBasicAuth(github_username, github_password)).text) -+ -+ if not data_: -+ break -+ -+ for data in data_: -+ try: -+ contributor = data['commit']['committer'] -+ contributor_email = contributor['email'] -+ contributor_fullname = contributor['name'] -+ contributor_name = data['committer']['login'] -+ except TypeError: -+ print 'Maybe one of the contributors is dropped because of lack of details' -+ continue -+ -+ json_data = { -+ 'name': contributor_name, -+ 'fullname': contributor_fullname, -+ 'emails': [contributor_email] -+ } -+ -+ present = False -+ for i in contributors: -+ if i == json_data: -+ present = True -+ break -+ -+ if not present: -+ print 'contributor added: ', contributor_name -+ contributors.append(json_data) -+ -+ with open('contributors.json', 'w') as f: -+ f.write(json.dumps(contributors)) -+ -+ return -+ -+ -+def generate_json_for_github_issue_commentors(github_username, github_password, \ -+ github_project_name): -+ ''' Will create a json file containing details of all the user -+ who have commented on any issue in the given project -+ ''' -+ -+ github_obj = Github(github_username, github_password) -+ project = github_obj.get_repo(github_project_name) -+ issue_comment_url = project.issue_comment_url.replace('{/number}', '') -+ -+ page = 0 -+ issue_commentors = [] -+ while True: -+ page += 1 -+ payload = {'page': page } -+ data_ = json.loads(requests.get(issue_comment_url, params=payload, -+ auth=HTTPBasicAuth(github_username, github_password)).text) -+ -+ if not data_: -+ break -+ -+ for data in data_: -+ try: -+ commentor = data['user']['login'] -+ except TypeError: -+ print 'Maybe one of the issue commentors have been dropped because of lack of details' -+ continue -+ -+ present = False -+ for i in issue_commentors: -+ if i == commentor: -+ present = True -+ break -+ -+ if not present: -+ print 'commentor added: ', commentor -+ issue_commentors.append(commentor) -+ -+ with open('issue_commentors.json', 'w') as f: -+ f.write(json.dumps(issue_commentors)) -+ return -+ -+ -+def assemble_github_contributors_commentors(): -+ ''' It uses the files: issue_commentors.json and contributors.json -+ Assembles and creates a file: assembled_commentors.csv -+ To use: just fill the empty blocks under emails column''' -+ -+ with open('issue_commentors.json', 'r') as ic: -+ issue_names = json.load(ic) -+ -+ with open('contributors.json', 'r') as c: -+ contributors = json.load(c) -+ -+ names = [] -+ for i in issue_names: -+ found = False -+ for j in contributors: -+ if j.get('name', None) == i: -+ j['emails'] = j.get('emails')[0] -+ names.append(j) -+ found = True -+ -+ if not found: -+ d = {'name': i, 'fullname': None, 'emails': None} -+ names.append(d) -+ -+ with open('assembled_commentors.csv', 'w') as ac: -+ field_names = ['name', 'fullname', 'emails'] -+ writer = csv.DictWriter(ac, fieldnames=field_names) -+ -+ writer.writeheader() -+ for name in names: -+ writer.writerow(name) -+ -+ -+def github_get_commentor_email(name): -+ ''' Will return the issue commentor email as given in the -+ assembled_commentors.csv file -+ ''' -+ -+ if not os.path.exists('assembled_commentors.csv'): -+ raise FileNotFound('The assembled_commentors.json file must be present \ -+ Rerun the program and choose to generate the json files') -+ -+ data = [] -+ with open('assembled_commentors.csv') as ac: -+ reader = csv.DictReader(ac) -+ for row in reader: -+ data.append(dict( \ -+ (('name', row['name']), \ -+ ('fullname', row['fullname']), \ -+ ('emails', row['emails'])))) -+ -+ -+ for i in data: -+ if i.get('name', None) == name: -+ if i['emails']: -+ return str(i['emails']) -+ else: -+ raise EmailNotFound('You need to fill out all the emails of the \ -+ issue commentors') -diff --git a/pagure_importer/utils/exceptions.py b/pagure_importer/utils/exceptions.py -new file mode 100644 -index 0000000..a5ab5fe ---- /dev/null -+++ b/pagure_importer/utils/exceptions.py -@@ -0,0 +1,22 @@ -+class GithubBadCredentials(Exception): -+ ''' Raised when username/password for github is wrong ''' -+ def __init__(self, msg): -+ self.msg = msg -+ -+ -+class GithubRepoNotFound(Exception): -+ ''' Raised when the repo is not found for the user ''' -+ def __init__(self, msg): -+ self.msg = msg -+ -+ -+class FileNotFound(Exception): -+ ''' Raised when a certain file is not found ''' -+ def __init__(self, msg): -+ self.msg = msg -+ -+ -+class EmailNotFound(Exception): -+ ''' Raised when email is not found ''' -+ def __init__(self, msg): -+ self.msg = msg -diff --git a/pagure_importer/utils/fas.py b/pagure_importer/utils/fas.py -new file mode 100644 -index 0000000..f62b5f5 ---- /dev/null -+++ b/pagure_importer/utils/fas.py -@@ -0,0 +1,23 @@ -+from fedora.client.fas2 import AccountSystem -+from pagure_importer.utils.models import User -+ -+ -+class FASclient (): -+ def __init__(self, fas_username, fas_password, fas_url): -+ self.fasclient = AccountSystem(fas_url, username=fas_username, -+ password=fas_password) -+ -+ anonymous = User(name='', fullname='', emails=[]) -+ self.fasuser = {'': anonymous} -+ -+ def find_fas_user(self, user): -+ -+ if user not in self.fasuser.keys(): -+ person = self.fasclient.person_by_username(user) -+ if not person: -+ return self.fasuser[''] -+ -+ self.fasuser[user] = User(name=user, -+ fullname=person['human_name'], -+ emails=[person['email']]) -+ return self.fasuser[user] -diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py -new file mode 100644 -index 0000000..33eba48 ---- /dev/null -+++ b/pagure_importer/utils/git.py -@@ -0,0 +1,98 @@ -+''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py -+ by pingou@pingoured.fr -+''' -+ -+import shutil -+import os -+import pygit2 -+import tempfile -+import json -+ -+from repo import * -+ -+def update_git(obj, repo_path, repo_folder): -+ """ Update the given issue in its git. -+ This method forks the provided repo, add/edit the issue whose file name -+ is defined by the uid field of the issue and if there are additions/ -+ changes commit them and push them back to the original repo. -+ """ -+ -+ if not repo_folder: -+ return -+ -+ # Get the fork -+ repopath = os.path.join(repo_folder, repo_path) -+ -+ # Clone the repo into a temp folder -+ newpath = tempfile.mkdtemp(prefix='pagure-') -+ new_repo = pygit2.clone_repository(repopath, newpath) -+ -+ file_path = os.path.join(newpath, obj.uid) -+ -+ # Get the current index -+ index = new_repo.index -+ -+ # Are we adding files -+ added = False -+ if not os.path.exists(file_path): -+ added = True -+ -+ # Write down what changed -+ with open(file_path, 'w') as stream: -+ stream.write(json.dumps( -+ obj.to_json(), sort_keys=True, indent=4, -+ separators=(',', ': '))) -+ -+ # Retrieve the list of files that changed -+ diff = new_repo.diff() -+ files = [] -+ for p in diff: -+ if hasattr(p, 'new_file_path'): -+ files.append(p.new_file_path) -+ elif hasattr(p, 'delta'): -+ files.append(p.delta.new_file.path) -+ -+ # Add the changes to the index -+ if added: -+ index.add(obj.uid) -+ for filename in files: -+ index.add(filename) -+ -+ # If not change, return -+ if not files and not added: -+ shutil.rmtree(newpath) -+ return -+ -+ # See if there is a parent to this commit -+ parent = None -+ try: -+ parent = new_repo.head.get_object().oid -+ except pygit2.GitError: -+ pass -+ -+ parents = [] -+ if parent: -+ parents.append(parent) -+ -+ # Author/commiter will always be this one -+ author = pygit2.Signature(name='pagure', email='pagure') -+ -+ # Actually commit -+ new_repo.create_commit( -+ 'refs/heads/master', -+ author, -+ author, -+ 'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title), -+ new_repo.index.write_tree(), -+ parents) -+ index.write() -+ -+ # Push to origin -+ ori_remote = new_repo.remotes[0] -+ master_ref = new_repo.lookup_reference('HEAD').resolve() -+ refname = '%s:%s' % (master_ref.name, master_ref.name) -+ -+ PagureRepo.push(ori_remote, refname) -+ -+ # Remove the clone -+ shutil.rmtree(newpath) -diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py -new file mode 100644 -index 0000000..cc61370 ---- /dev/null -+++ b/pagure_importer/utils/importer_github.py -@@ -0,0 +1,138 @@ -+from github import Github -+ -+from pagure_importer.utils import models -+from pagure_importer.utils import github_get_commentor_email -+from pagure_importer.utils.git import update_git -+from pagure_importer.utils.exceptions import ( -+ GithubBadCredentials, -+ GithubRepoNotFound -+) -+ -+ -+class GithubImporter(): -+ ''' Imports from Github using PyGithub and libpagure ''' -+ def __init__( -+ self, -+ github_username, -+ github_password, -+ github_project_name): -+ self.github_username = github_username -+ self.github_password = github_password -+ self.github_project_name = github_project_name -+ self.github = Github(github_username, github_password) -+ -+ def import_issues(self, repo_path, repo_folder, status='all'): -+ ''' Imports the issues on github for -+ the given project -+ ''' -+ github_user = None -+ try: -+ github_user = self.github.get_user(self.github_username) -+ except: -+ raise GithubBadCredentials( -+ 'Given github credentials are not correct') -+ repo = self.github.get_repo(self.github_project_name) -+ try: -+ repo_name = repo.name -+ except: -+ raise GithubRepoNotFound( -+ 'Repo not found, project name wrong') -+ -+ for github_issue in repo.get_issues(state=status): -+ -+ # title of the issue -+ pagure_issue_title = github_issue.title -+ -+ # body of the issue -+ if github_issue.body: -+ pagure_issue_content = github_issue.body -+ else: -+ pagure_issue_content = '#No Description Provided' -+ -+ # Some details of a issue -+ if github_issue.state != 'closed': -+ pagure_issue_status = 'Open' -+ else: -+ pagure_issue_status = 'Fixed' -+ -+ pagure_issue_created_at = github_issue.created_at -+ -+ # Not sure how to deal with this atm -+ pagure_issue_assignee = None -+ -+ if github_issue.labels: -+ pagure_issue_tags = [i.name for i in github_issue.labels] -+ else: -+ pagure_issue_tags = [] -+ -+ -+ # few things not supported by github -+ pagure_issue_depends = [] -+ pagure_issue_blocks = [] -+ pagure_issue_is_private = False -+ -+ -+ # User who created the issue -+ pagure_issue_user = models.User( -+ name=github_issue.user.login, -+ fullname=github_issue.user.name, -+ emails=[github_issue.user.email]) -+ -+ -+ pagure_issue = models.Issue( -+ id=None, -+ title = pagure_issue_title, -+ content = pagure_issue_content, -+ status = pagure_issue_status, -+ date_created = pagure_issue_created_at, -+ user = pagure_issue_user.to_json(), -+ private = pagure_issue_is_private, -+ tags = pagure_issue_tags, -+ depends = pagure_issue_depends, -+ blocks = pagure_issue_blocks, -+ assignee = pagure_issue_assignee) -+ -+ -+ # comments on the issue -+ comments = [] -+ for comment in github_issue.get_comments(): -+ -+ comment_user = comment.user -+ pagure_issue_comment_user_email = comment_user.email -+ pagure_issue_comment_body = comment.body -+ pagure_issue_comment_created_at = comment.created_at -+ pagure_issue_comment_updated_at = comment.updated_at -+ -+ -+ # No idea what to do with this right now -+ # editor: not supported by github api -+ pagure_issue_comment_parent = None -+ pagure_issue_comment_editor = None -+ -+ # comment updated at -+ pagure_issue_comment_edited_on = comment.updated_at -+ -+ # The User who commented -+ pagure_issue_comment_user = models.User( -+ name=comment_user.login, -+ fullname=comment_user.name, -+ emails=[comment_user.email] if comment_user.email \ -+ else [github_get_commentor_email(comment_user.login)]) -+ -+ # Object to represent comment on an issue -+ pagure_issue_comment = models.IssueComment( -+ id=None, -+ comment=pagure_issue_comment_body, -+ parent=pagure_issue_comment_parent, -+ date_created=pagure_issue_comment_created_at, -+ user=pagure_issue_comment_user.to_json(), -+ edited_on=pagure_issue_comment_edited_on, -+ editor=pagure_issue_comment_editor) -+ -+ comments.append(pagure_issue_comment.to_json()) -+ -+ # add all the comments to the issue object -+ pagure_issue.comments = comments -+ -+ # update the local git repo -+ update_git(pagure_issue, repo_path, repo_folder) -diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py -new file mode 100644 -index 0000000..50f20b0 ---- /dev/null -+++ b/pagure_importer/utils/importer_trac.py -@@ -0,0 +1,35 @@ -+from xmlrpclib import ServerProxy -+from pagure_importer.utils.git import update_git -+from pagure_importer.utils import trac -+ -+ -+class TracImporter(): -+ '''Pagure importer for trac instance''' -+ -+ def __init__(self, trac_project_url, fasclient=None): -+ self.tracclient = ServerProxy(trac_project_url + '/rpc') -+ if fasclient: -+ self.fasclient = fasclient -+ -+ def import_issues(self, repo_path, repo_folder, -+ trac_query='max=0&order=id'): -+ '''Import issues from trac instance using xmlrpc API''' -+ tickets_id = self.tracclient.ticket.query(trac_query) -+ -+ for ticket_id in tickets_id: -+ -+ pagure_issue = trac.populate_issue(self.tracclient, -+ self.fasclient, ticket_id) -+ -+ pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) -+ comments = trac.populate_comments(self.fasclient, -+ pagure_issue_comments) -+ -+ # add all the comments to the issue object -+ pagure_issue.comments = comments -+ -+ # update the local git repo -+ print 'Update repo with issue :' + str(ticket_id) -+ update_git(pagure_issue, -+ repo_path, -+ repo_folder) -diff --git a/pagure_importer/utils/models.py b/pagure_importer/utils/models.py -new file mode 100644 -index 0000000..3aa3d83 ---- /dev/null -+++ b/pagure_importer/utils/models.py -@@ -0,0 +1,105 @@ -+# -*- coding: utf-8 -*- -+ -+import datetime -+import json -+import uuid -+ -+class Issue(): -+ ''' Represents an Issue ''' -+ -+ def __init__( -+ self, id, title, content, -+ status, date_created, user, private, tags, -+ depends, blocks, assignee, comments=None): -+ -+ self.id = id -+ self.title = title -+ self.content = content -+ self.status = status -+ self.date_created = date_created -+ self.user = user -+ self.private = private -+ self.tags = tags -+ self.depends = depends -+ self.blocks = blocks -+ self.assignee = assignee -+ self.comments = comments -+ self.uid = uuid.uuid4().hex -+ -+ def to_json(self): -+ ''' Returns a dictionary representation of the issue. -+ -+ ''' -+ output = { -+ 'id': self.id, -+ 'title': self.title, -+ 'content': self.content, -+ 'status': self.status, -+ 'date_created': self.date_created.strftime('%s'), -+ 'user': self.user, -+ 'private': self.private, -+ 'tags': self.tags, -+ 'depends': self.depends, -+ 'blocks': self.blocks, -+ 'assignee': self.assignee, -+ 'comments': self.comments -+ } -+ -+ return output -+ -+ @property -+ def isa(self): -+ return 'issue' -+ -+ -+class IssueComment(): -+ ''' Represent a comment for an issue ''' -+ -+ def __init__( -+ self, id, comment, date_created, -+ user, parent=None, edited_on=None, editor=None): -+ -+ self.id = id -+ self.comment = comment -+ self.parent = parent -+ self.date_created = date_created -+ self.user = user -+ self.edited_on = edited_on -+ self.editor = editor -+ -+ def to_json(self): -+ ''' Returns a dictionary representation of the issue. ''' -+ -+ output = { -+ 'id': self.id, -+ 'comment': self.comment, -+ 'parent': self.parent, -+ 'date_created': self.date_created.strftime('%s'), -+ 'user': self.user, -+ 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, -+ 'editor': self.editor or None -+ } -+ -+ return output -+ -+ -+class User(): -+ ''' Represents a User ''' -+ -+ def __init__( -+ self, name, emails, -+ fullname=None): -+ self.name = name -+ self.fullname = fullname -+ self.emails = emails -+ -+ def to_json(self): -+ ''' Return a representation of the User in a dictionary. ''' -+ -+ output = { -+ 'name': self.name, -+ 'fullname': self.fullname, -+ 'emails': self.emails -+ } -+ -+ return output -diff --git a/pagure_importer/utils/repo.py b/pagure_importer/utils/repo.py -new file mode 100644 -index 0000000..f46b31a ---- /dev/null -+++ b/pagure_importer/utils/repo.py -@@ -0,0 +1,70 @@ -+# -*- coding: utf-8 -*- -+ -+''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py -+ by pingou@pingoured.fr -+''' -+ -+ -+import pygit2 -+import sys -+ -+ -+def get_pygit2_version(): -+ ''' Return pygit2 version as a tuple of integers. -+ This is needed for correct version comparison. -+ ''' -+ return tuple([int(i) for i in pygit2.__version__.split('.')]) -+ -+ -+class PagureRepo(pygit2.Repository): -+ """ An utility class allowing to go around pygit2's inability to be -+ stable. -+ -+ """ -+ -+ @staticmethod -+ def push(remote, refname): -+ """ Push the given reference to the specified remote. """ -+ pygit2_version = get_pygit2_version() -+ if pygit2_version >= (0, 22): -+ remote.push([refname]) -+ else: -+ remote.push(refname) -+ -+ def pull(self, remote_name='origin', branch='master', force=False): -+ ''' pull changes for the specified remote (defaults to origin). -+ -+ Code from MichaelBoselowitz at: -+ https://github.com/MichaelBoselowitz/pygit2-examples/blob/ -+ 68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58 -+ licensed under the MIT license. -+ ''' -+ -+ for remote in self.remotes: -+ if remote.name == remote_name: -+ remote.fetch() -+ remote_master_id = self.lookup_reference( -+ 'refs/remotes/origin/%s' % branch).target -+ -+ if force: -+ repo_branch = self.lookup_reference( -+ 'refs/heads/%s' % branch) -+ repo_branch.set_target(remote_master_id) -+ -+ merge_result, _ = self.merge_analysis(remote_master_id) -+ # Up to date, do nothing -+ if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: -+ return -+ # We can just fastforward -+ elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: -+ self.checkout_tree(self.get(remote_master_id)) -+ master_ref = self.lookup_reference( -+ 'refs/heads/%s' % branch) -+ master_ref.set_target(remote_master_id) -+ self.head.set_target(remote_master_id) -+ elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: -+ sys.exit('Pulling remote changes leads to a conflict') -+ else: -+ print 'Unexpected merge result: %s' % ( -+ pygit2.GIT_MERGE_ANALYSIS_NORMAL) -+ raise AssertionError('Unknown merge analysis result') -diff --git a/pagure_importer/utils/trac.py b/pagure_importer/utils/trac.py -new file mode 100644 -index 0000000..d56a123 ---- /dev/null -+++ b/pagure_importer/utils/trac.py -@@ -0,0 +1,90 @@ -+from pagure_importer.utils.models import IssueComment, Issue -+from datetime import datetime -+ -+ -+def get_ticket_tags(trac_ticket): -+ return [] -+ -+ -+def get_ticket_status(trac_ticket): -+ ''' Converts Trac ticket status -+ to Pagure issue status''' -+ -+ if trac_ticket['status'] != 'closed': -+ ticket_status = 'Open' -+ else: -+ ticket_status = 'Fixed' -+ return ticket_status -+ -+ -+def populate_comments(fasclient, trac_comments): -+ comments = [] -+ for comment in trac_comments: -+ if comment[2] == 'comment' and comment[4] != '': -+ comment_user = comment[1] -+ pagure_issue_comment_user_email = None -+ pagure_issue_comment_body = comment[4] -+ pagure_issue_comment_created_at = datetime.strptime( -+ comment[0].value, "%Y%m%dT%H:%M:%S") -+ pagure_issue_comment_updated_at = None -+ -+ # No idea what to do with this right now -+ # editor: not supported by github api -+ pagure_issue_comment_parent = None -+ pagure_issue_comment_editor = None -+ -+ # comment updated at -+ pagure_issue_comment_edited_on = None -+ -+ # The User who commented -+ pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) -+ -+ # Object to represent comment on an issue -+ pagure_issue_comment = IssueComment( -+ id=None, -+ comment=pagure_issue_comment_body, -+ parent=pagure_issue_comment_parent, -+ date_created=pagure_issue_comment_created_at, -+ user=pagure_issue_comment_user.to_json(), -+ edited_on=pagure_issue_comment_edited_on, -+ editor=pagure_issue_comment_editor) -+ -+ comments.append(pagure_issue_comment.to_json()) -+ return comments -+ -+ -+def populate_issue(trac, fasclient, ticket_id): -+ trac_ticket = trac.ticket.get(ticket_id)[3] -+ pagure_issue_title = trac_ticket['summary'] -+ pagure_issue_content = trac_ticket['description'] -+ -+ if pagure_issue_content == '': -+ pagure_issue_content = '#No Description Provided' -+ -+ pagure_issue_status = get_ticket_status(trac_ticket) -+ -+ pagure_issue_created_at = datetime.strptime( -+ trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") -+ -+ pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) -+ -+ pagure_issue_tags = get_ticket_tags(trac_ticket) -+ -+ pagure_issue_depends = [] -+ pagure_issue_blocks = [] -+ pagure_issue_is_private = False -+ -+ pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) -+ pagure_issue = Issue( -+ id=ticket_id, -+ title=pagure_issue_title, -+ content=pagure_issue_content, -+ status=pagure_issue_status, -+ date_created=pagure_issue_created_at, -+ user=pagure_issue_user.to_json(), -+ private=pagure_issue_is_private, -+ tags=pagure_issue_tags, -+ depends=pagure_issue_depends, -+ blocks=pagure_issue_blocks, -+ assignee=pagure_issue_assignee.to_json()) -+ return pagure_issue -diff --git a/setup.py b/setup.py -index e031405..65fa0d5 100644 ---- a/setup.py -+++ b/setup.py -@@ -32,7 +32,7 @@ setup( - license='GNU General Public License v2.0', - entry_points={ - 'console_scripts': [ -- 'pgimport = pagure_importer.run:main' -+ 'pgimport = pagure_importer.app:app' - ], - }, - include_package_data=True, diff --git a/requirements.txt b/requirements.txt index 52fc450..e966d85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,5 @@ PyGithub requests +click +python-fedora +pygit2 >= 0.20.1 From 98445b30c3794779bf1b5404bdc265022e3e006e Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 04 2016 20:33:19 +0000 Subject: [PATCH 8/24] Added usage to README + fix requirements.txt --- diff --git a/README.md b/README.md index b2075e2..0d3d02b 100644 --- a/README.md +++ b/README.md @@ -2,25 +2,54 @@ CLI tool for importing issues etc. from different sources like github to pagure ## Installation +--- 1. Install it using ```pip``` . ```pip install pagure_importer``` ## How to run +--- 0. Clone the issue tracker for issues from pagure. Use: ```git clone --bare``` 1. set the env variables: ```REPO_NAME``` and ```REPO_PATH``` ex: REPO_NAME=abc.git; REPO_PATH=/home/vivek/ 2. Activate the pagure tickets hook from project settings. -3. Execute ```pgimport``` +3. Execute ```pgimport```. See Usage section 4. Just answer what is asked. Check below instructions for particular source 5. The script will make commits in your cloned bare repo: push the changes back to pagure. -### Present options for sources: github -### Present options for items: issues + +## Usage +----- + + + $ pgimport --help + Usage: pgimport [OPTIONS] COMMAND [ARGS]... + + Options: + --help Show this message and exit. + + Commands: + fedorahosted + github + + +The fedorahosted command can be used to import issues from a fedorahosted project to pagure + + $ pgimport fedorahosted https://fedorahosted.org/fedocal + + +The github command can be used to import issues from a github project to pagure + + $ pgimport github + ### Tools used: +--- 1. [PyGithub](https://github.com/PyGithub/PyGithub) - a python library for [github](https://github.com/) api. +2. [click](https://github.com/pallets/click) - Python package for creating beautiful command line interfaces +3. [python-fedora](https://fedorahosted.org/python-fedora/) - A collection of python code that allows programs to talk to Fedora Services ## How it works: Github Issues +--- 0. For github issues, there is a bit of pre-processing so, the process is not very user friendly. The reason behind the pre-processing is that: github doesn't give away the email ids of issue commentors unless the commentor diff --git a/requirements.txt b/requirements.txt index e966d85..ba8d9e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ PyGithub -requests click python-fedora pygit2 >= 0.20.1 From c76ad225d967731ebee6d3f7b4d50b225a43a6c6 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 08 2016 08:25:32 +0000 Subject: [PATCH 9/24] removed debug comment + fix README separation line --- diff --git a/README.md b/README.md index 0d3d02b..4e8c209 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ ex: REPO_NAME=abc.git; REPO_PATH=/home/vivek/ ## Usage ------ +--- $ pgimport --help diff --git a/pagure_importer/app.py b/pagure_importer/app.py index 6dc3277..7c92a53 100644 --- a/pagure_importer/app.py +++ b/pagure_importer/app.py @@ -15,7 +15,6 @@ __all__ = [ 'app', ] -# from .commands import github from .commands import fedorahosted from .commands import github From e0c21e9c98dac76c4ca7a81fb5a81c5523bd0726 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 09 2016 17:00:47 +0000 Subject: [PATCH 10/24] Display the number of issue to imported --- diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index 8e7fff8..9fdcd14 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -7,7 +7,7 @@ class TracImporter(): '''Pagure importer for trac instance''' def __init__(self, trac_project_url, fasclient=None): - self.tracclient = ServerProxy(trac_project_url + '/rpc') + self.tracclient = ServerProxy(trac_project_url) self.fasclient = fasclient def import_issues(self, repo_path, repo_folder, @@ -28,5 +28,6 @@ class TracImporter(): pagure_issue.comments = comments # update the local git repo - print 'Update repo with issue :' + str(ticket_id) + print 'Update repo with issue :' + str(ticket_id) + '/' +\ + str(tickets_id[-1]) update_git(pagure_issue, repo_path, repo_folder) From 8fd92641c46e8b28f549627f5417743a7e5867c4 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 09 2016 17:04:59 +0000 Subject: [PATCH 11/24] Use FAS username and password to to perform authenticated calls of the XML-RPC APIs --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 9ca8a53..15fc1c4 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -11,5 +11,10 @@ def fedorahosted(project_url): fas_password = getpass.getpass('Enter your FAS password: ') fasclient = FASclient(fas_username, fas_password, 'https://admin.fedoraproject.org/accounts') - trac_importer = importer_trac.TracImporter(project_url, fasclient) + + rpc_login = fas_username + ':' + fas_password + '@' + url_index = project_url.find('://') + rpc_url = project_url[:url_index+3] + rpc_login + project_url[url_index+3:]\ + + '/login/xmlrpc' + trac_importer = importer_trac.TracImporter(rpc_url, fasclient) trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) From 4a6b28b480a90ce7b9a4a46517a1710cf1987311 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 10 2016 22:20:03 +0000 Subject: [PATCH 12/24] Added basic support for issue tags --- diff --git a/pagure_importer/utils/trac.py b/pagure_importer/utils/trac.py index 4847959..5abb015 100644 --- a/pagure_importer/utils/trac.py +++ b/pagure_importer/utils/trac.py @@ -2,10 +2,6 @@ from pagure_importer.utils.models import IssueComment, Issue, User from datetime import datetime -def get_ticket_tags(trac_ticket): - return [] - - def get_ticket_status(trac_ticket): ''' Converts Trac ticket status to Pagure issue status''' @@ -79,7 +75,15 @@ def populate_issue(trac, fasclient, ticket_id): pagure_issue_assignee = anonymous pagure_issue_user = anonymous - pagure_issue_tags = get_ticket_tags(trac_ticket) + pagure_issue_tags = [] + if trac_ticket['type'] != '': + pagure_issue_tags.append(trac_ticket['type']) + if trac_ticket['milestone'] != '': + pagure_issue_tags.append(trac_ticket['milestone']) + if trac_ticket['component'] != '': + pagure_issue_tags.append(trac_ticket['component']) + if trac_ticket['version'] != '': + pagure_issue_tags.append(trac_ticket['version']) pagure_issue_depends = [] pagure_issue_blocks = [] From 9af8efeab8e3962da81356b3aa16d2009caf4740 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 13 2016 19:45:06 +0000 Subject: [PATCH 13/24] Added Import pagure tags option to fedorahosted command If flag is set then we import the tags, otherwise tags will be left empty --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 15fc1c4..403d269 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -6,7 +6,8 @@ from pagure_importer.utils.fas import FASclient @app.command() @click.argument('project_url') -def fedorahosted(project_url): +@click.option('--tags', help="Import pagure tags:", is_flag=True) +def fedorahosted(project_url, tags): fas_username = raw_input('Enter you FAS Username: ') fas_password = getpass.getpass('Enter your FAS password: ') fasclient = FASclient(fas_username, fas_password, @@ -17,4 +18,5 @@ def fedorahosted(project_url): rpc_url = project_url[:url_index+3] + rpc_login + project_url[url_index+3:]\ + '/login/xmlrpc' trac_importer = importer_trac.TracImporter(rpc_url, fasclient) - trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) + trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH, + tags=tags) diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index 9fdcd14..deb2cb8 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -10,7 +10,7 @@ class TracImporter(): self.tracclient = ServerProxy(trac_project_url) self.fasclient = fasclient - def import_issues(self, repo_path, repo_folder, + def import_issues(self, repo_path, repo_folder, tags, trac_query='max=0&order=id'): '''Import issues from trac instance using xmlrpc API''' tickets_id = self.tracclient.ticket.query(trac_query) @@ -18,7 +18,7 @@ class TracImporter(): for ticket_id in tickets_id: pagure_issue = trac.populate_issue(self.tracclient, - self.fasclient, ticket_id) + self.fasclient, ticket_id, tags) pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) comments = trac.populate_comments(self.fasclient, diff --git a/pagure_importer/utils/trac.py b/pagure_importer/utils/trac.py index 5abb015..b5e131b 100644 --- a/pagure_importer/utils/trac.py +++ b/pagure_importer/utils/trac.py @@ -54,7 +54,7 @@ def populate_comments(fasclient, trac_comments): return comments -def populate_issue(trac, fasclient, ticket_id): +def populate_issue(trac, fasclient, ticket_id, tags): trac_ticket = trac.ticket.get(ticket_id)[3] pagure_issue_title = trac_ticket['summary'] pagure_issue_content = trac_ticket['description'] @@ -76,14 +76,15 @@ def populate_issue(trac, fasclient, ticket_id): pagure_issue_user = anonymous pagure_issue_tags = [] - if trac_ticket['type'] != '': - pagure_issue_tags.append(trac_ticket['type']) - if trac_ticket['milestone'] != '': - pagure_issue_tags.append(trac_ticket['milestone']) - if trac_ticket['component'] != '': - pagure_issue_tags.append(trac_ticket['component']) - if trac_ticket['version'] != '': - pagure_issue_tags.append(trac_ticket['version']) + if tags: + if trac_ticket['type'] != '': + pagure_issue_tags.append(trac_ticket['type']) + if trac_ticket['milestone'] != '': + pagure_issue_tags.append(trac_ticket['milestone']) + if trac_ticket['component'] != '': + pagure_issue_tags.append(trac_ticket['component']) + if trac_ticket['version'] != '': + pagure_issue_tags.append(trac_ticket['version']) pagure_issue_depends = [] pagure_issue_blocks = [] From a07755a2cd0b94ac63f3cca667bcff71cbba1497 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 29 2016 20:45:14 +0000 Subject: [PATCH 14/24] Added clone command to pgimport Command enables user to clone a ticket repo using the url provided by pagure --- diff --git a/pagure_importer/app.py b/pagure_importer/app.py index 7c92a53..9523ffe 100644 --- a/pagure_importer/app.py +++ b/pagure_importer/app.py @@ -17,6 +17,7 @@ __all__ = [ from .commands import fedorahosted from .commands import github +from .commands import clone if __name__ == '__main__': app() diff --git a/pagure_importer/commands/clone.py b/pagure_importer/commands/clone.py new file mode 100644 index 0000000..86ac1b8 --- /dev/null +++ b/pagure_importer/commands/clone.py @@ -0,0 +1,14 @@ +import click +import os +import subprocess as sp +from pagure_importer.app import app + +@app.command() +@click.argument('repo_url') +def clone (repo_url): + repo = os.path.join('/tmp/',repo_url.split('/')[-1]) + cmd = ['git', 'clone', '--bare',repo_url, repo] + proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.STDOUT) + output, _ = proc.communicate() + output = output.decode('utf-8') + click.echo(output) From 8503f08cd7d555c2296698f2cedbfbdb957bd616 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 29 2016 20:47:12 +0000 Subject: [PATCH 15/24] Modify fedorahosted import to diplay the ticket repo found in the tmp folder --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 403d269..253aa08 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -1,9 +1,12 @@ import click +import os import getpass -from pagure_importer.app import app, REPO_NAME, REPO_PATH -from pagure_importer.utils import importer_trac +from pagure_importer.app import app, REPO_NAME +from pagure_importer.utils import importer_trac, display_repo from pagure_importer.utils.fas import FASclient +REPO_PATH='/tmp/' + @app.command() @click.argument('project_url') @click.option('--tags', help="Import pagure tags:", is_flag=True) @@ -17,6 +20,8 @@ def fedorahosted(project_url, tags): url_index = project_url.find('://') rpc_url = project_url[:url_index+3] + rpc_login + project_url[url_index+3:]\ + '/login/xmlrpc' + display_repo() + repo_name = raw_input('Choose the import destination repo : ') trac_importer = importer_trac.TracImporter(rpc_url, fasclient) - trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH, + trac_importer.import_issues(repo_path=repo_name, repo_folder=REPO_PATH, tags=tags) diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 05ccdb8..5afed11 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -11,6 +11,13 @@ from requests.auth import HTTPBasicAuth from pagure_importer.utils.exceptions import FileNotFound, EmailNotFound +def display_repo(): + print '#### Repo available ####' + for file in os.listdir('/tmp/'): + if file.endswith('.git'): + print ' * ' + file + print + def generate_json_for_github_contributors(github_username, github_password, \ github_project_name): ''' Creates a file containing a list of dicts containing the username and From bf38ad9a888e8cb71abcfc94dc6d87391bf93757 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 29 2016 21:11:06 +0000 Subject: [PATCH 16/24] Removed dependency to /tmp folder and used tempfile lib Fixed pep8 code syntax --- diff --git a/pagure_importer/app.py b/pagure_importer/app.py index 9523ffe..303088f 100644 --- a/pagure_importer/app.py +++ b/pagure_importer/app.py @@ -1,10 +1,9 @@ #!/usr/bin/env python - import click -import os +import tempfile -REPO_NAME = os.environ.get('REPO_NAME', None) # this has to be a bare repo -REPO_PATH = os.environ.get('REPO_PATH', None) # the parent of the git directory +REPO_NAME = '' +REPO_PATH = tempfile.gettempdir() @click.group() diff --git a/pagure_importer/commands/clone.py b/pagure_importer/commands/clone.py index 86ac1b8..b6eb03a 100644 --- a/pagure_importer/commands/clone.py +++ b/pagure_importer/commands/clone.py @@ -1,13 +1,14 @@ import click import os import subprocess as sp -from pagure_importer.app import app +from pagure_importer.app import app, REPO_PATH + @app.command() @click.argument('repo_url') -def clone (repo_url): - repo = os.path.join('/tmp/',repo_url.split('/')[-1]) - cmd = ['git', 'clone', '--bare',repo_url, repo] +def clone(repo_url): + repo = os.path.join(REPO_PATH, repo_url.split('/')[-1]) + cmd = ['git', 'clone', '--bare', repo_url, repo] proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.STDOUT) output, _ = proc.communicate() output = output.decode('utf-8') diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 253aa08..8955160 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -1,11 +1,10 @@ import click import os import getpass -from pagure_importer.app import app, REPO_NAME -from pagure_importer.utils import importer_trac, display_repo +from pagure_importer.app import app, REPO_PATH +from pagure_importer.utils import importer_trac from pagure_importer.utils.fas import FASclient -REPO_PATH='/tmp/' @app.command() @click.argument('project_url') @@ -14,13 +13,13 @@ def fedorahosted(project_url, tags): fas_username = raw_input('Enter you FAS Username: ') fas_password = getpass.getpass('Enter your FAS password: ') fasclient = FASclient(fas_username, fas_password, - 'https://admin.fedoraproject.org/accounts') + 'https://admin.fedoraproject.org/accounts') rpc_login = fas_username + ':' + fas_password + '@' url_index = project_url.find('://') - rpc_url = project_url[:url_index+3] + rpc_login + project_url[url_index+3:]\ - + '/login/xmlrpc' - display_repo() + rpc_url = project_url[:url_index+3] + rpc_login +\ + project_url[url_index+3:] + '/login/xmlrpc' + pagure_importer.utils.display_repo() repo_name = raw_input('Choose the import destination repo : ') trac_importer = importer_trac.TracImporter(rpc_url, fasclient) trac_importer.import_issues(repo_path=repo_name, repo_folder=REPO_PATH, diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 5afed11..5f3b2ae 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -10,16 +10,20 @@ from github import Github from requests.auth import HTTPBasicAuth from pagure_importer.utils.exceptions import FileNotFound, EmailNotFound +from pagure_importer.app import REPO_PATH + def display_repo(): - print '#### Repo available ####' - for file in os.listdir('/tmp/'): + print '#### Repo available ####' + for file in os.listdir(REPO_PATH): if file.endswith('.git'): print ' * ' + file print -def generate_json_for_github_contributors(github_username, github_password, \ - github_project_name): + +def generate_json_for_github_contributors(github_username, + github_password, + github_project_name): ''' Creates a file containing a list of dicts containing the username and emails of the contributors in the given github project ''' @@ -32,7 +36,7 @@ def generate_json_for_github_contributors(github_username, github_password, \ contributors = [] while True: page += 1 - payload = {'page': page } + payload = {'page': page} data_ = json.loads(requests.get(commits_url, params=payload, auth=HTTPBasicAuth(github_username, github_password)).text) @@ -71,8 +75,9 @@ def generate_json_for_github_contributors(github_username, github_password, \ return -def generate_json_for_github_issue_commentors(github_username, github_password, \ - github_project_name): +def generate_json_for_github_issue_commentors(github_username, + github_password, + github_project_name): ''' Will create a json file containing details of all the user who have commented on any issue in the given project ''' @@ -85,7 +90,7 @@ def generate_json_for_github_issue_commentors(github_username, github_password, issue_commentors = [] while True: page += 1 - payload = {'page': page } + payload = {'page': page} data_ = json.loads(requests.get(issue_comment_url, params=payload, auth=HTTPBasicAuth(github_username, github_password)).text) @@ -160,12 +165,11 @@ def github_get_commentor_email(name): with open('assembled_commentors.csv') as ac: reader = csv.DictReader(ac) for row in reader: - data.append(dict( \ - (('name', row['name']), \ - ('fullname', row['fullname']), \ + data.append(dict( + (('name', row['name']), + ('fullname', row['fullname']), ('emails', row['emails'])))) - for i in data: if i.get('name', None) == name: if i['emails']: From 7dc2b1c8092146c3004b2cec25375f713ec7d8af Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 30 2016 09:54:55 +0000 Subject: [PATCH 17/24] Removed REPO_NAME env variable from github import --- diff --git a/pagure_importer/app.py b/pagure_importer/app.py index 303088f..680de1c 100644 --- a/pagure_importer/app.py +++ b/pagure_importer/app.py @@ -2,7 +2,6 @@ import click import tempfile -REPO_NAME = '' REPO_PATH = tempfile.gettempdir() diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py index 26ae4fa..13f80e7 100644 --- a/pagure_importer/commands/github.py +++ b/pagure_importer/commands/github.py @@ -1,7 +1,7 @@ import click import getpass -from pagure_importer.app import app, REPO_NAME, REPO_PATH +from pagure_importer.app import app, REPO_PATH from pagure_importer.utils.importer_github import GithubImporter from pagure_importer.utils import ( generate_json_for_github_contributors, @@ -16,17 +16,21 @@ def form_github_issues(): github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') return (github_username, github_password, github_project_name) + @app.command() def github(): github_username, github_password, github_project_name = form_github_issues() gen_json = raw_input( - 'Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') + 'Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') if gen_json == 'n': github_importer = GithubImporter( github_username=github_username, github_password=github_password, github_project_name=github_project_name) - github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) + + pagure_importer.utils.display_repo() + repo_name = raw_input('Choose the import destination repo : ') + github_importer.import_issues(repo_path=repo_name, repo_folder=REPO_PATH) else: generate_json_for_github_contributors( github_username, From aa5af06cc390c096651bd09bbd127339c5e02d64 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 30 2016 10:53:45 +0000 Subject: [PATCH 18/24] Updated README with pgimport clone usage --- diff --git a/README.md b/README.md index 4e8c209..acd5132 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,11 @@ CLI tool for importing issues etc. from different sources like github to pagure ## How to run --- -0. Clone the issue tracker for issues from pagure. Use: ```git clone --bare``` -1. set the env variables: ```REPO_NAME``` and ```REPO_PATH``` -ex: REPO_NAME=abc.git; REPO_PATH=/home/vivek/ -2. Activate the pagure tickets hook from project settings. -3. Execute ```pgimport```. See Usage section -4. Just answer what is asked. Check below instructions for particular source -5. The script will make commits in your cloned bare repo: push the changes back to pagure. +0. Clone the issue tracker for issues from pagure. Use: ```pgimport clone ssh://git@pagure.io/tickets/foobar.git``` +1. Activate the pagure tickets hook from project settings. +2. Execute ```pgimport```. See Usage section +3. Just answer what is asked. Check below instructions for particular source +4. The script will make commits in your cloned bare repo: push the changes back to pagure. ## Usage @@ -27,13 +25,26 @@ ex: REPO_NAME=abc.git; REPO_PATH=/home/vivek/ --help Show this message and exit. Commands: + clone fedorahosted github +The clone command can be used to clone the pagure ticket repository: + + $ pgimport clone ssh://git@pagure.io/tickets/foobar.git + The fedorahosted command can be used to import issues from a fedorahosted project to pagure + + $ pgimport fedorahosted --help + Usage: pgimport fedorahosted [OPTIONS] PROJECT_URL + + Options: + --tags Import pagure tags: + --help Show this message and exit. + - $ pgimport fedorahosted https://fedorahosted.org/fedocal + $ pgimport fedorahosted https://fedorahosted.org/foobar The github command can be used to import issues from a github project to pagure From 70a4f8df8fd8604a32654953849c0cc4dcc91acf Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 30 2016 11:48:24 +0000 Subject: [PATCH 19/24] Fixed missing import for pagure_importer.utils.display_repo() --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 8955160..9c50992 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -1,6 +1,7 @@ import click import os import getpass +import pagure_importer from pagure_importer.app import app, REPO_PATH from pagure_importer.utils import importer_trac from pagure_importer.utils.fas import FASclient From b248628a85be74bdd931252d75b0c661632a8099 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 30 2016 11:48:50 +0000 Subject: [PATCH 20/24] Added push command to push back temp ticket repo to pagure --- diff --git a/pagure_importer/app.py b/pagure_importer/app.py index 680de1c..1c2b507 100644 --- a/pagure_importer/app.py +++ b/pagure_importer/app.py @@ -16,6 +16,7 @@ __all__ = [ from .commands import fedorahosted from .commands import github from .commands import clone +from .commands import push if __name__ == '__main__': app() diff --git a/pagure_importer/commands/push.py b/pagure_importer/commands/push.py new file mode 100644 index 0000000..97b9705 --- /dev/null +++ b/pagure_importer/commands/push.py @@ -0,0 +1,16 @@ +import click +import os +import subprocess as sp +from pagure_importer.app import app, REPO_PATH + + +@app.command() +@click.argument('repo_name') +def push (repo_name): + repo = os.path.join(REPO_PATH, repo_name) + os.chdir(repo) + cmd = ['git', 'push', 'origin', 'master' ] + proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.STDOUT) + output, _ = proc.communicate() + output = output.decode('utf-8') + click.echo(output) From b0fcbe932f2353ef001d1cf41e22a7fde6e1152d Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 30 2016 12:16:06 +0000 Subject: [PATCH 21/24] Updated pgimport push usage in README --- diff --git a/README.md b/README.md index acd5132..6aa91fb 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ CLI tool for importing issues etc. from different sources like github to pagure 1. Activate the pagure tickets hook from project settings. 2. Execute ```pgimport```. See Usage section 3. Just answer what is asked. Check below instructions for particular source -4. The script will make commits in your cloned bare repo: push the changes back to pagure. +4. The script will make commits in your cloned repo: push the changes back to pagure. Use : ```pgimport push foobar.git``` ## Usage @@ -28,6 +28,7 @@ CLI tool for importing issues etc. from different sources like github to pagure clone fedorahosted github + push The clone command can be used to clone the pagure ticket repository: @@ -51,6 +52,10 @@ The github command can be used to import issues from a github project to pagure $ pgimport github +The push command can be used to push a clone pagure ticket repo back to pagure. + + $ pgimport push foobar.git + ### Tools used: --- From b9b29571cd77e876c0147311aaa9d00498762398 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 31 2016 21:36:06 +0000 Subject: [PATCH 22/24] Cleanup the ticket repo display function. Ask the user to enter a number rather than the repo name. --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 9c50992..44c234a 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -20,8 +20,12 @@ def fedorahosted(project_url, tags): url_index = project_url.find('://') rpc_url = project_url[:url_index+3] + rpc_login +\ project_url[url_index+3:] + '/login/xmlrpc' - pagure_importer.utils.display_repo() - repo_name = raw_input('Choose the import destination repo : ') - trac_importer = importer_trac.TracImporter(rpc_url, fasclient) - trac_importer.import_issues(repo_path=repo_name, repo_folder=REPO_PATH, - tags=tags) + repos = pagure_importer.utils.display_repo() + if repos: + repo_index = raw_input('Choose the import destination repo (default 1) : ') or 1 + repo_name = repos[int(repo_index)-1] + trac_importer = importer_trac.TracImporter(rpc_url, fasclient) + trac_importer.import_issues(repo_name=repo_name, repo_folder=REPO_PATH, + tags=tags) + else: + click.echo('No ticket repository found. Use pgimport clone command') diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 5f3b2ae..94750fd 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -14,11 +14,16 @@ from pagure_importer.app import REPO_PATH def display_repo(): + repo = [] + index = 0 print '#### Repo available ####' for file in os.listdir(REPO_PATH): if file.endswith('.git'): - print ' * ' + file + index += 1 + print str(index) + ' - ' + file + repo.append(file) print + return repo def generate_json_for_github_contributors(github_username, diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index deb2cb8..b0ea1b5 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -10,7 +10,7 @@ class TracImporter(): self.tracclient = ServerProxy(trac_project_url) self.fasclient = fasclient - def import_issues(self, repo_path, repo_folder, tags, + def import_issues(self, repo_name, repo_folder, tags, trac_query='max=0&order=id'): '''Import issues from trac instance using xmlrpc API''' tickets_id = self.tracclient.ticket.query(trac_query) @@ -28,6 +28,6 @@ class TracImporter(): pagure_issue.comments = comments # update the local git repo - print 'Update repo with issue :' + str(ticket_id) + '/' +\ + print 'Update ' + repo_name + ' with issue :' + str(ticket_id) + '/' +\ str(tickets_id[-1]) - update_git(pagure_issue, repo_path, repo_folder) + update_git(pagure_issue, repo_name, repo_folder) From 82e01c6e52e5d1deb6fc9e5d666fee1dd4b5e9f1 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Jun 13 2016 18:21:15 +0000 Subject: [PATCH 23/24] Fix github importer to use new cli command pgimport clone and pgimport push --- diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py index 13f80e7..5aae121 100644 --- a/pagure_importer/commands/github.py +++ b/pagure_importer/commands/github.py @@ -1,6 +1,6 @@ import click import getpass - +import pagure_importer from pagure_importer.app import app, REPO_PATH from pagure_importer.utils.importer_github import GithubImporter from pagure_importer.utils import ( @@ -28,9 +28,14 @@ def github(): github_password=github_password, github_project_name=github_project_name) - pagure_importer.utils.display_repo() - repo_name = raw_input('Choose the import destination repo : ') - github_importer.import_issues(repo_path=repo_name, repo_folder=REPO_PATH) + repos = pagure_importer.utils.display_repo() + if repos: + repo_index = raw_input('Choose the import destination repo (default 1) : ') or 1 + repo_name = repos[int(repo_index)-1] + github_importer.import_issues(repo_path=repo_name, repo_folder=REPO_PATH) + else: + click.echo('No ticket repository found. Use pgimport clone command') + else: generate_json_for_github_contributors( github_username, From 010f24a90e4409d48a28eda643eaed49c6417e3c Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Jun 13 2016 18:24:12 +0000 Subject: [PATCH 24/24] Update version to 1.0.0 --- diff --git a/setup.py b/setup.py index 65fa0d5..686164f 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ def read(fname): setup( name='pagure_importer', packages=['pagure_importer'], - version='0.0.3', + version='1.0.0', description='CLI tool for imports to Pagure', author='Vivek Anand', author_email='vivekanand1101@gmail.com',