From 6df4a6be8f46f1236fe77f18a1be00a7f5c7b526 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Mar 27 2016 20:07:54 +0000 Subject: [PATCH 1/8] Added argparse support for CLI. Option added: Source, Item and Match --- diff --git a/pagure_importer/run.py b/pagure_importer/run.py old mode 100644 new mode 100755 index a485493..52f91c0 --- a/pagure_importer/run.py +++ b/pagure_importer/run.py @@ -1,28 +1,40 @@ #!/usr/bin/env python - +from argparse import ArgumentParser from forms import form_github_issues from settings import IMPORT_SOURCES, IMPORT_OPTIONS -def github_handler(item): - if item.lower() == 'issues': - form_github_issues() + +def import_handler(source, item, match): + # TODO - manage match flag in forms or importer class. + if item == 'issues': + if source.lower() == 'github': + form_github_issues() + elif source == 'fedorahosted': + form_fedorahosted() return + def main(): - source = raw_input('Enter source from where you want to import: ') - if source.lower() not in IMPORT_SOURCES: - print 'Source location not supported' - return + parser = ArgumentParser(description='Import existing project to pagure\ + from different sources') + parser.add_argument('source', metavar='src', + help='sources: github, fedorahosted') + parser.add_argument('-i', '--item', default='issues', + help='items to be imported. default: issues') + parser.add_argument('-m', '--match', + help="Fail the import is pagure issue number doesn't match source", + action="store_true") + args = parser.parse_args() - item = raw_input('Enter the item to be imported: ') - if item.lower() not in IMPORT_OPTIONS[source]: - print 'Item import not supported' + if args.source.lower() not in IMPORT_SOURCES: + print 'Source not supported (pgimport -h for help)' return - if source.lower() == 'github': - github_handler(item) + if args.item.lower() not in IMPORT_OPTIONS[args.source]: + print 'Item not supported (pgimport -h for help)' + return - return + import_handler(args.source.lower(), args.item.lower(), args.match) if __name__ == '__main__': main() diff --git a/pagure_importer/settings.py b/pagure_importer/settings.py index 13f8332..c92226f 100644 --- a/pagure_importer/settings.py +++ b/pagure_importer/settings.py @@ -1,2 +1,2 @@ -IMPORT_SOURCES = ['github'] -IMPORT_OPTIONS = {'github': ['issues']} +IMPORT_SOURCES = ['github', 'fedorahosted'] +IMPORT_OPTIONS = {'github': ['issues'], 'fedorahosted': ['issues'] } From 1f08ad6ac21fb6cc1e7626c7762037eeec13e086 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Mar 27 2016 20:44:14 +0000 Subject: [PATCH 2/8] Added Added new class to manage specific pagure settings (importer_helper). Each source specific importer will inherit from this. --- diff --git a/pagure_importer/sources/importer_github.py b/pagure_importer/sources/importer_github.py index cc85f7b..38435f7 100644 --- a/pagure_importer/sources/importer_github.py +++ b/pagure_importer/sources/importer_github.py @@ -1,14 +1,14 @@ import libpagure from libpagure.libpagure import Pagure +from importer_helper import ImporterHelper from github import Github from exceptions import GithubBadCredentials, GithubRepoNotFound -class GithubImporter(): +class GithubImporter(ImporterHelper): ''' Imports from Github using PyGithub and libpagure ''' - def __init__( self, github_username, @@ -19,35 +19,14 @@ class GithubImporter(): pagure_username=None, instance_url='https://pagure.io'): + super(GithubImporter, self).__init__(pagure_api_key, + pagure_project_name, + pagure_username, + instance_url) self.github_username = github_username self.github_password = github_password self.github_project_name = github_project_name - self.pagure_project_name = pagure_project_name self.github = Github(github_username, github_password) - self.pagure = Pagure(pagure_api_key, pagure_project_name, - pagure_username, instance_url) - - - def _get_available_issue_id(self): - ''' Private method which checks the id - which would be available for the new issue - ''' - issues = self.pagure.list_issues() - pull_requests = self.pagure.list_requests() - max_issues = None - max_pull_requests = None - try: - max_issues = max([int(issue['id']) for issue in issues]) - except ValueError: - max_issues = 0 - - try: - max_pull_requests = max([int(pr['id']) for pr in pull_requests]) - except ValueError: - max_pull_requests = 0 - - return max(max_issues, max_pull_requests) + 1 - def _get_repo(self, github_user): ''' Private method to get the repo object @@ -58,8 +37,7 @@ class GithubImporter(): if repo.name == self.github_project_name: return repo raise GithubRepoNotFound( - 'No user repository with given github project name found') - + 'No user repository with given github project name found') def import_issues(self, status='all'): ''' Imports the issues on github for @@ -70,7 +48,7 @@ class GithubImporter(): github_user = self.github.get_user(self.github_username) except: raise GithubBadCredentials( - 'Given github credentials are not correct') + 'Given github credentials are not correct') repo = self._get_repo(github_user) for github_issue in repo.get_issues(state=status): @@ -82,13 +60,12 @@ class GithubImporter(): issue_id = self._get_available_issue_id() self.pagure.create_issue( - pagure_issue_title, pagure_issue_content) + pagure_issue_title, pagure_issue_content) - #comments on the issue + # comments on the issue for comment in github_issue.get_comments(): self.pagure.comment_issue(issue_id, str(comment.body)) - #change status of the issue if closed + # change status of the issue if closed if github_issue.state.lower() == 'closed': self.pagure.change_issue_status(issue_id, 'Fixed') - diff --git a/pagure_importer/sources/importer_helper.py b/pagure_importer/sources/importer_helper.py new file mode 100644 index 0000000..9eaee62 --- /dev/null +++ b/pagure_importer/sources/importer_helper.py @@ -0,0 +1,29 @@ +import libpagure +from libpagure.libpagure import Pagure + + +class ImporterHelper(object): + """ Manages setting for pagure APIs """ + def __init__(self, + pagure_api_key, + pagure_project_name, + pagure_username=None, + instance_url='https://pagure.io'): + + self.pagure = Pagure(pagure_api_key, pagure_project_name, + pagure_username, instance_url) + + def _get_available_issue_id(self): + ''' Private method which checks the id + which would be available for the new issue + ''' + + issues = self.pagure.list_issues(status='all') + max_issues = None + + try: + max_issues = max([int(issue['id']) for issue in issues]) + except ValueError: + max_issues = 0 + + return max_issues + 1 From b554bdcc59b25d28dfa4687d11124d643223aaaf Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Mar 27 2016 20:49:39 +0000 Subject: [PATCH 3/8] added class to support fedorahosted --- diff --git a/pagure_importer/sources/importer_fedorahosted.py b/pagure_importer/sources/importer_fedorahosted.py new file mode 100644 index 0000000..448815d --- /dev/null +++ b/pagure_importer/sources/importer_fedorahosted.py @@ -0,0 +1,47 @@ +import libpagure +from libpagure.libpagure import Pagure +from importer_helper import ImporterHelper +from xmlrpclib import ServerProxy + + +class TracImporter(ImporterHelper): + ''' Imports issues from Trac XML-RPC API to pagure''' + + def __init__(self, + trac_project_name, + pagure_api_key, + pagure_project_name, + pagure_username, + instance_url='https://pagure.io'): + + super(GithubImporter, self).__init__(pagure_api_key, + pagure_project_name, + pagure_username, + instance_url) + self.trac = ServerProxy('https://fedorahosted.org/' + + trac_project_name + '/rpc') + + def import_issues(self, trac_query='report=9&order=id'): + + tickets_list = [] + tickets_id = self.trac.ticket.query(trac_query) + for ticket_id in tickets_id: + trac_ticket = self.trac.ticket.get(ticket_id)[3] + pagure_issue_content = trac_ticket['description'] + pagure_issue_title = trac_ticket['summary'] + pagure_issue_status = trac_ticket['status'] + pagure_issue_comments = self.trac.ticket.changeLog(ticket_id) + + if pagure_issue_content == '': + pagure_issue_content = 'No Description' + + issue_id = self._get_available_issue_id() + + self.pagure.create_issue(pagure_issue_title, pagure_issue_content) + + for comment in pagure_issue_comments: + if comment[2] == 'comment' and comment[4] != '': + self.pagure.comment_issue(issue_id, comment[4]) + + if pagure_issue_status == 'closed': + self.pagure.change_issue_status(issue_id, 'Fixed') From d95a2d177d59d487ca5fcff1b773ebeefd365b84 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Mar 30 2016 17:25:31 +0000 Subject: [PATCH 4/8] rework forms for fedorahosted --- diff --git a/pagure_importer/forms.py b/pagure_importer/forms.py index 1aa883a..bcfe0f3 100644 --- a/pagure_importer/forms.py +++ b/pagure_importer/forms.py @@ -1,35 +1,62 @@ from sources.importer_github import GithubImporter +from sources.importer_fedorahosted import TracImporter import getpass -def form_github_issues(): - github_username = raw_input('Enter you Github Username: ') - github_password = getpass.getpass('Enter your github password: ') - github_project_name = raw_input('Enter github project name: ') + +def form_helper(): pagure_api_key = raw_input('Enter your pagure api key: ') pagure_project_name = raw_input('Enter pagure project name: ') - is_forked = raw_input('Is the pagure project a forked repo ? (y/n): ') or 'n' + if is_forked.lower() == 'y': pagure_username = raw_input('Enter your pagure username: ') else: pagure_username = None is_pagure_io = raw_input( - 'Is the pagure instance url - https://pagure.io ?: (y/n) ') or 'y' + 'Is the pagure instance url - https://pagure.io ?: (y/n) ') or 'y' if is_pagure_io.lower() == 'n': - pagure_instance = raw_input('Enter the pagure instance url: ') or 'https://pagure.io' + pagure_instance = raw_input('Enter the pagure instance url: ')\ + or 'https://pagure.io' else: pagure_instance = 'https://pagure.io' status = raw_input( - 'Enter status of the issues to be imported (all/open/closed): ') or 'all' + 'Enter status of the issues to be imported (all/open/closed): ') or 'all' + + +def form_github_issues(): + form_helper() + github_username = raw_input('Enter you Github Username: ') + github_password = getpass.getpass('Enter your github password: ') + github_project_name = raw_input('Enter github project name: ') github_importer = GithubImporter( - github_username=github_username, - github_password=github_password, - github_project_name=github_project_name, - pagure_api_key=pagure_api_key, - pagure_project_name=pagure_project_name, - pagure_username=pagure_username, - instance_url=pagure_instance) + github_username=github_username, + github_password=github_password, + github_project_name=github_project_name, + pagure_api_key=pagure_api_key, + pagure_project_name=pagure_project_name, + pagure_username=pagure_username, + instance_url=pagure_instance) github_importer.import_issues(status) + + +def form_fedorahosted_issues(): + form_helper() + fedorahosted_project_name = raw_input('Enter fedorahosted project name: ') + + if status == 'all': + status = 'report=9&order=id' + elif status == 'open': + status = 'status=new&order=id' + elif status == 'closed': + status = 'status=closed&order=id' + + fedorahosted_importer = TracImporter( + trac_project_name=fedorahosted_project_name, + pagure_api_key=pagure_api_'status=new&order=id'key, + pagure_project_name=pagure_project_name, + pagure_username=pagure_username, + instance_url=pagure_instance) + fedorahosted_importer.import_issues(status) From 70212349ec9c96051489e1b1ab27fcb15ae12a66 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Mar 31 2016 08:08:49 +0000 Subject: [PATCH 5/8] Fixing runtime errors --- diff --git a/pagure_importer/forms.py b/pagure_importer/forms.py index bcfe0f3..4819e02 100644 --- a/pagure_importer/forms.py +++ b/pagure_importer/forms.py @@ -24,9 +24,13 @@ def form_helper(): status = raw_input( 'Enter status of the issues to be imported (all/open/closed): ') or 'all' + return (pagure_api_key, pagure_project_name, pagure_username, + pagure_instance, status) + def form_github_issues(): - form_helper() + (pagure_api_key, pagure_project_name, pagure_username, + pagure_instance, status) = form_helper() github_username = raw_input('Enter you Github Username: ') github_password = getpass.getpass('Enter your github password: ') github_project_name = raw_input('Enter github project name: ') @@ -43,7 +47,8 @@ def form_github_issues(): def form_fedorahosted_issues(): - form_helper() + (pagure_api_key, pagure_project_name, pagure_username, + pagure_instance, status) = form_helper() fedorahosted_project_name = raw_input('Enter fedorahosted project name: ') if status == 'all': @@ -55,7 +60,7 @@ def form_fedorahosted_issues(): fedorahosted_importer = TracImporter( trac_project_name=fedorahosted_project_name, - pagure_api_key=pagure_api_'status=new&order=id'key, + pagure_api_key=pagure_api_key, pagure_project_name=pagure_project_name, pagure_username=pagure_username, instance_url=pagure_instance) diff --git a/pagure_importer/run.py b/pagure_importer/run.py index 52f91c0..decb676 100755 --- a/pagure_importer/run.py +++ b/pagure_importer/run.py @@ -1,6 +1,6 @@ #!/usr/bin/env python from argparse import ArgumentParser -from forms import form_github_issues +from forms import form_github_issues, form_fedorahosted_issues from settings import IMPORT_SOURCES, IMPORT_OPTIONS @@ -10,7 +10,7 @@ def import_handler(source, item, match): if source.lower() == 'github': form_github_issues() elif source == 'fedorahosted': - form_fedorahosted() + form_fedorahosted_issues() return diff --git a/pagure_importer/sources/importer_fedorahosted.py b/pagure_importer/sources/importer_fedorahosted.py index 448815d..b99c4e9 100644 --- a/pagure_importer/sources/importer_fedorahosted.py +++ b/pagure_importer/sources/importer_fedorahosted.py @@ -14,7 +14,7 @@ class TracImporter(ImporterHelper): pagure_username, instance_url='https://pagure.io'): - super(GithubImporter, self).__init__(pagure_api_key, + super(TracImporter, self).__init__(pagure_api_key, pagure_project_name, pagure_username, instance_url) @@ -22,7 +22,9 @@ class TracImporter(ImporterHelper): trac_project_name + '/rpc') def import_issues(self, trac_query='report=9&order=id'): - + ''' Imports the issues from fedorahosted for + the given project. Default imports all issues. + ''' tickets_list = [] tickets_id = self.trac.ticket.query(trac_query) for ticket_id in tickets_id: From 5696201d8a1f71c193447de59d81354abcd1f4cd Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Mar 31 2016 08:10:52 +0000 Subject: [PATCH 6/8] fixing pep8 issues --- diff --git a/pagure_importer/sources/importer_fedorahosted.py b/pagure_importer/sources/importer_fedorahosted.py index b99c4e9..3a9b9a3 100644 --- a/pagure_importer/sources/importer_fedorahosted.py +++ b/pagure_importer/sources/importer_fedorahosted.py @@ -15,9 +15,9 @@ class TracImporter(ImporterHelper): instance_url='https://pagure.io'): super(TracImporter, self).__init__(pagure_api_key, - pagure_project_name, - pagure_username, - instance_url) + pagure_project_name, + pagure_username, + instance_url) self.trac = ServerProxy('https://fedorahosted.org/' + trac_project_name + '/rpc') From d8c1b09b567c0a9afbe782a98192abd856b12907 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Mar 31 2016 08:19:16 +0000 Subject: [PATCH 7/8] Updated README and setup to release 0.0.4 --- diff --git a/README.md b/README.md index b0c883d..7297361 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,11 @@ CLI tool for importing issues etc. from different sources like github to pagure ## How to run 1. Install it using ```pip``` . ```pip install pagure_importer``` -2. Execute ```pgimport``` -3. Just answer what is asked, one by one. +2. Execute ```pgimport -h``` for usage. +3. Github import example : ```pgimport github``` +4. Then just answer what is asked, one by one. +5. Fedorahosted import example : ```pgimport fedorahosted``` +6. Then just answer what is asked, one by one. ### Present options for sources: github ### Present options for items: issues diff --git a/setup.py b/setup.py index e031405..0ccc602 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ def read(fname): setup( name='pagure_importer', packages=['pagure_importer'], - version='0.0.3', + version='0.0.4', description='CLI tool for imports to Pagure', author='Vivek Anand', author_email='vivekanand1101@gmail.com', From b8ca3b77d2b18adb45f05ef32366ef9ae743743a Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Apr 01 2016 19:51:07 +0000 Subject: [PATCH 8/8] Fix _get_available_issue_id to include PRs --- diff --git a/pagure_importer/sources/importer_helper.py b/pagure_importer/sources/importer_helper.py index 9eaee62..0ab3819 100644 --- a/pagure_importer/sources/importer_helper.py +++ b/pagure_importer/sources/importer_helper.py @@ -19,11 +19,17 @@ class ImporterHelper(object): ''' issues = self.pagure.list_issues(status='all') + pull_requests = self.pagure.list_requests(status='all') max_issues = None - + max_pull_requests = None try: max_issues = max([int(issue['id']) for issue in issues]) except ValueError: max_issues = 0 - return max_issues + 1 + try: + max_pull_requests = max([int(pr['id']) for pr in pull_requests]) + except ValueError: + max_pull_requests = 0 + + return max(max_issues, max_pull_requests) + 1