From c63e119e6de2d46612ebaa5c147d69ad41aecd71 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Aug 21 2016 10:11:18 +0000 Subject: [PATCH 1/7] Replaced xml-rpc login url to use json rpc requests --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index e11d794..f241c2c 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -10,20 +10,17 @@ from pagure_importer.utils.fas import FASclient @click.argument('project_url') @click.option('--tags', help="Import pagure tags:", is_flag=True) def fedorahosted(project_url, tags): - fas_username = raw_input('Enter you FAS Username: ') - fas_password = getpass.getpass('Enter your FAS password: ') - fasclient = FASclient(fas_username, fas_password, + username = raw_input('Enter you FAS Username: ') + password = getpass.getpass('Enter your FAS password: ') + fasclient = FASclient(username, password, 'https://admin.fedoraproject.org/accounts') - - rpc_login = fas_username + ':' + fas_password + '@' - url_index = project_url.find('://') - rpc_url = project_url[:url_index+3] + rpc_login +\ - project_url[url_index+3:] + '/login/xmlrpc' + project_url = project_url + '/login/jsonrpc' repos = pagure_importer.utils.display_repo() if repos: repo_index = raw_input('Choose the import destination repo (default 1) : ') or 1 repo_name = repos[int(repo_index)-1] - trac_importer = importer_trac.TracImporter(rpc_url, fasclient, tags) + trac_importer = importer_trac.TracImporter(project_url, username, + password, fasclient, tags) trac_importer.import_issues(repo_name=repo_name, repo_folder=REPO_PATH) else: click.echo('No ticket repository found. Use pgimport clone command') From a174c25bec1ed4d6eb0bab74cc9ce3130be09dae Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Aug 21 2016 10:41:41 +0000 Subject: [PATCH 2/7] Started using http requests for json rpc instead of xml rpc --- diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index a298bb3..d2c3cf3 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -1,4 +1,4 @@ -from xmlrpclib import ServerProxy +import requests from datetime import datetime from pagure_importer.utils.git import update_git, get_secure_filename from pagure_importer.utils.models import User, Issue, IssueComment @@ -7,16 +7,35 @@ from pagure_importer.utils.models import User, Issue, IssueComment class TracImporter(): '''Pagure importer for trac instance''' - def __init__(self, trac_project_url, fasclient=None, tags=False): - self.trac = ServerProxy(trac_project_url) + def __init__(self, project_url, username, password, fasclient=None, tags=False): + self.url = project_url + self.username = username + self.password = password self.fas = fasclient self.tags = tags self.somebody = User(name='somebody', fullname='somebody', emails=['some@body.com']) + self.reqid = 0 + + def request(self, method, *args): + self.reqid += 1 + req = {'params': args, + 'method': method, + 'id': self.reqid} + resp = requests.post(self.url, json=req, auth=(self.username, self.password)) + resp = resp.json() + if resp['id'] != self.reqid: + print('ERROR: Invalid response for request! ID does not match') + sys.exit(1) + if resp['error'] != None: + print("ERROR: Error in response: %s" % resp['error']) + sys.exit(1) + + return resp['result'] def import_issues(self, repo_name, repo_folder, trac_query='max=0&order=id'): '''Import issues from trac instance using xmlrpc API''' - tickets_id = self.trac.ticket.query(trac_query) + tickets_id = self.request('ticket.query', trac_query) for ticket_id in tickets_id: pagure_issue = self.create_issue(ticket_id) @@ -39,13 +58,13 @@ class TracImporter(): def create_issue(self, ticket_id): - trac_ticket = self.trac.ticket.get(ticket_id)[3] - trac_attachments = self.trac.ticket.listAttachments(ticket_id) + trac_ticket = self.request('ticket.get', ticket_id)[3] + trac_attachments = self.request('ticket.listAttachments', ticket_id) pagure_attachment = {} for attachment in trac_attachments: filename = attachment[0] - content = self.trac.ticket.getAttachment(ticket_id, filename) + content = self.request('ticket.getAttachment', ticket_id, filename) pagure_attachment[filename] = content pagure_issue_title = trac_ticket['summary'] From fab2e17e7de3bfd6eb91af8bfaae21f872df4c12 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Aug 21 2016 14:45:00 +0000 Subject: [PATCH 3/7] Trac importer using jsonrpc --- diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index d2c3cf3..439ba35 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -1,4 +1,5 @@ import requests +import time from datetime import datetime from pagure_importer.utils.git import update_git, get_secure_filename from pagure_importer.utils.models import User, Issue, IssueComment @@ -32,6 +33,12 @@ class TracImporter(): return resp['result'] + def to_timestamp(self, tm): + tm = tm.replace('+00:00', '') + date = datetime.strptime(tm, '%Y-%m-%dT%H:%M:%S') + ts = str(time.mktime(date.timetuple()))[:-2] # Strip the .0 + return ts + def import_issues(self, repo_name, repo_folder, trac_query='max=0&order=id'): '''Import issues from trac instance using xmlrpc API''' @@ -40,7 +47,7 @@ class TracImporter(): for ticket_id in tickets_id: pagure_issue = self.create_issue(ticket_id) pagure_issue.comments = [] - pagure_issue_comments = self.trac.ticket.changeLog(ticket_id) + pagure_issue_comments = self.request('ticket.changeLog', ticket_id) comments = self.create_comments(pagure_issue_comments) # add all the comments to the issue object for key in comments: @@ -75,8 +82,7 @@ class TracImporter(): pagure_issue_status = self.get_ticket_status(trac_ticket) - pagure_issue_created_at = datetime.strptime( - self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") + pagure_issue_created_at = self.to_timestamp(self.request('ticket.get', ticket_id)[1]['__jsonclass__'][1]) if self.fas: pagure_issue_assignee = self.fas.find_fas_user(trac_ticket['owner']) @@ -132,7 +138,7 @@ class TracImporter(): def create_comments(self, trac_comments): comments = {} for comment in trac_comments: - ts = datetime.strptime(comment[0].value, "%Y%m%dT%H:%M:%S") + ts = self.to_timestamp(comment[0]['__jsonclass__'][1]) if comment[2] == 'comment' and comment[4] != '': if ts in comments: diff --git a/pagure_importer/utils/models.py b/pagure_importer/utils/models.py index 3c129d6..1e18710 100644 --- a/pagure_importer/utils/models.py +++ b/pagure_importer/utils/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import datetime import json import uuid @@ -36,7 +35,7 @@ class Issue(): 'title': self.title, 'content': self.content, 'status': self.status, - 'date_created': self.date_created.strftime('%s'), + 'date_created': self.date_created, 'user': self.user, 'private': self.private, 'tags': self.tags, @@ -79,9 +78,9 @@ class IssueComment(): 'id': self.id, 'comment': self.comment, 'parent': self.parent, - 'date_created': self.date_created.strftime('%s'), + 'date_created': self.date_created, 'user': self.user, - 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, + 'edited_on': self.edited_on if self.edited_on else None, 'editor': self.editor or None } From ac3cd227ef495b0da79808f526e29d8ed46bd18d Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Aug 23 2016 20:19:36 +0000 Subject: [PATCH 4/7] Fix attachment and reduce number of request to optimise runtime --- diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index 439ba35..e6055e2 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -1,5 +1,6 @@ import requests import time +import base64 from datetime import datetime from pagure_importer.utils.git import update_git, get_secure_filename from pagure_importer.utils.models import User, Issue, IssueComment @@ -65,14 +66,15 @@ class TracImporter(): def create_issue(self, ticket_id): - trac_ticket = self.request('ticket.get', ticket_id)[3] + trac_ticket_info = self.request('ticket.get', ticket_id) + trac_ticket = trac_ticket_info[3] trac_attachments = self.request('ticket.listAttachments', ticket_id) pagure_attachment = {} for attachment in trac_attachments: filename = attachment[0] - content = self.request('ticket.getAttachment', ticket_id, filename) - pagure_attachment[filename] = content + content = self.request('ticket.getAttachment', ticket_id, filename)['__jsonclass__'][1].replace('\n', '') + pagure_attachment[filename] = base64.b64decode(content) pagure_issue_title = trac_ticket['summary'] @@ -82,7 +84,7 @@ class TracImporter(): pagure_issue_status = self.get_ticket_status(trac_ticket) - pagure_issue_created_at = self.to_timestamp(self.request('ticket.get', ticket_id)[1]['__jsonclass__'][1]) + pagure_issue_created_at = self.to_timestamp(trac_ticket_info[1]['__jsonclass__'][1]) if self.fas: pagure_issue_assignee = self.fas.find_fas_user(trac_ticket['owner']) From 14ce7742c7b6002536659deea11b622a457aae40 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Aug 29 2016 06:52:20 +0000 Subject: [PATCH 5/7] Remove install from git as this is not working --- diff --git a/README.md b/README.md index 0ee4dd6..a04cda4 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,10 @@ # pagure-importer CLI tool for importing issues etc. from different sources like github to pagure -## Installation +## Installation in a Virtual Environment --- * Install it using ```pip``` . ```pip install pagure_importer``` - ## How to run --- 0. Clone the issue tracker for issues from pagure. Use: ```pgimport clone ssh://git@pagure.io/tickets/foobar.git``` @@ -15,18 +14,6 @@ CLI tool for importing issues etc. from different sources like github to pagure 4. The script will make commits in your cloned repo: push the changes back to pagure. Use : ```pgimport push foobar.git``` -## Run from git ---- - -* Install the dependencies: - - sudo dnf install python-github python-pygit2 python-fedora python-click python-werkzeug - -* Run the application - - PYTHONPATH=. python pagure_importer/app.py - - ## Usage --- From 0273adabd2657b4b1b8add0f3345d6897e29874e Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Aug 29 2016 06:52:45 +0000 Subject: [PATCH 6/7] Add requests library to requirements.txt --- diff --git a/requirements.txt b/requirements.txt index 780699f..9723285 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ click python-fedora pygit2 >= 0.20.1 werkzeug +requests From e7a2c65e9027de4b8817d2f6d1267e5968bdd515 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Aug 29 2016 09:59:57 +0000 Subject: [PATCH 7/7] Protect call to fas api is user is None --- diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index e6055e2..9c9fd7e 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -129,7 +129,7 @@ class TracImporter(): def get_comment_user(self, comment): # The User who commented - if self.fas: + if self.fas and comment[1]: pagure_issue_comment_user = self.fas.find_fas_user(comment[1]) if not pagure_issue_comment_user.name: pagure_issue_comment_user = self.somebody