From f1e5608c63e166de3192ee84b81c0e396ea0f13c Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Feb 07 2017 06:09:15 +0000 Subject: [PATCH 1/8] Update to use python3 in case installing from pip Signed-off-by: Vivek Anand --- diff --git a/README.md b/README.md index 92cba3c..69848ca 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ CLI tool for importing issues etc. from different sources like github to pagure --- #### In a Virtual Environment -* Install it using ```pip``` . ```pip install pagure_importer``` +* Install it using ```pip``` . ```python3 -m pip install pagure_importer``` #### Using COPR package * Install it using [copr](https://copr.fedorainfracloud.org/coprs/cverna/pagure-importer/). From bc70b1bd0ff1887fdf73aeda447e5a4f2c22359f Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Feb 08 2017 06:29:54 +0000 Subject: [PATCH 2/8] remove .git in case it's present in project_url Signed-off-by: Vivek Anand --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index b3b1986..5e22a64 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -24,6 +24,8 @@ def fedorahosted( project_url, tags, private, username, password, offset, nopush): fasclient = FASclient(username, password, 'https://admin.fedoraproject.org/accounts') + if project_url.endswith('.git'): + project_url = project_url.replace('.git', '') project_url += '/login/jsonrpc' repos = pagure_importer.utils.display_repo() if repos: From 0c72a9b27933c9b02626451c6a0acfa8efc9bf90 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 09 2017 09:48:56 +0000 Subject: [PATCH 3/8] Create a local user within the docker container The idea is that we are mounting the ssh keys from the user running the container into the container and in Fedora, that user will have the UID 1000, so these keys won't be available inside the container which is by default ran as root. So let's create a local user within the container, it will have the UID 1000 and thus mounting the ssh keys will work. Note that this will not work if the user running the container does not have the same UID. --- diff --git a/Dockerfile b/Dockerfile index d54779c..62e987b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,4 +5,6 @@ RUN dnf install -y dnf-plugins-core && \ dnf copr enable -y cverna/pagure-importer && \ dnf install -y python3-pgimport +RUN adduser -m duser + ENV LC_ALL=en_GB.UTF-8 From 4cd6618eb949b4ba329e36ef1f7c21e02e6a6255 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 09 2017 09:51:03 +0000 Subject: [PATCH 4/8] Adjust the docker run command The order of the argument matters (-v first, -i -t then) and provide the tag built with and to run as otherwise the -t argument is not used. --- diff --git a/README.md b/README.md index 69848ca..53b4b5d 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ CLI tool for importing issues etc. from different sources like github to pagure #### Using Docker * Build and run a container using the Dockerfile in this repository ``` - $ docker build -t pgimport . - $ docker run -t -i -v ~/.ssh/:/root/.ssh/:Z /bin/bash + $ sudo docker build -t pgimport . + $ sudo docker run -v ~/.ssh/:/home/duser/.ssh/:Z -i -t pgimport /bin/bash ``` * Inside the container you can use the pgimport command. From 75398708bf577654a18beb3508ffc3df770613d6 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Feb 12 2017 11:05:54 +0000 Subject: [PATCH 5/8] Beginning of refactor to remove TracImporter object Signed-off-by: Clement Verna --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 5e22a64..0f3004d 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -2,7 +2,6 @@ import click import pagure_importer from pagure_importer.app import app, REPO_PATH from pagure_importer.utils import importer_trac, get_pagure_namespace -from pagure_importer.utils.fas import FASclient import pagure_importer.utils.git as gitutils diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index ffaee9d..004a7bd 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -1,15 +1,20 @@ +import os import sys import re import time import click import requests -import shutil -import os +from random import randint from base64 import b64decode from datetime import datetime from pagure_importer.utils import ( get_close_status, is_image, issue_to_json, get_secure_filename) from pagure_importer.utils.models import User, Issue, IssueComment +from pagure_importer.utils.fas import FASclient +from pagure_importer.app import REPO_PATH + +SOMEBODY = User(name='somebody', fullname='somebody', + emails=['some@body.com']) def to_timestamp(tm): @@ -21,298 +26,276 @@ def to_timestamp(tm): return ts -class TracImporter(object): - ''' Pagure importer for trac instance ''' - - def __init__(self, project_url, username, password, offset, repo_name, - repo_folder, nopush, fasclient=None, tags=False, private=False): - ''' Instantiate a TracImporter object ''' - self.username = username - self.password = password - self.repo_name = repo_name - self.repo_folder = repo_folder - self.clone_repo_location = os.path.join( - repo_folder, 'clone-' + repo_name) - self.nopush = nopush - self.url = project_url - self.fas = fasclient - self.tags = tags - self.private = private - self.offset = offset - self.somebody = User(name='somebody', fullname='somebody', - emails=['some@body.com']) - self.reqid = 0 - self.custom_fields = [] - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - ''' Delete the cloned repo where the commits were going ''' - if os.path.exists(self.clone_repo_location): - if not self.nopush: - shutil.rmtree(self.clone_repo_location) - - def request(self, method, *args): - ''' Common method for querying trac ''' - - self.reqid += 1 - req = {'params': args, - 'method': method, - 'id': self.reqid} - resp = requests.post(self.url, json=req, - auth=(self.username, self.password)) - resp = resp.json() - if resp['id'] != self.reqid: - click.echo('ERROR: Invalid response for request! ' + - 'ID does not match') - sys.exit(1) - if resp['error'] is not None: - # Ignore missing attachment errors - if 'Attachment ' not in resp['error']['message'] and \ - ' not found' not in resp['error']['message']: - click.echo("ERROR: Error in response: %s" % resp['error']) - sys.exit(1) - - return resp['result'] - - def get_custom_fields(self): - ''' Queries the fedorahosted api to get all ticket fields - and filters all the custom fields, returns - a list of dicts - dict with keys 'name' and 'key_type' ''' - - all_ticket_fields = self.request('ticket.getTicketFields') - custom_fields = [] - for field in all_ticket_fields: - if field.get('custom') is True: - current_field = {} - current_field['name'] = field['name'] - key_type = 'text' - if field['type'] == 'checkbox': - key_type = 'boolean' - current_field['key_type'] = key_type - custom_fields.append(current_field) - return custom_fields - - def import_issues(self, repo_name, trac_query='max=0&order=id'): - ''' Queries the trac instance via its jsonrpc API and convert the - tickets into JSON blob to be imported into pagure's ticket git repo. - - :arg repo_name: the name of the repository - :arg repo_folder: the folder in which is the repository - :kwarg trac_query: the query to call trac with in order to retrieve - all the tickets. - Defaults to ``max=0&order=id`` - - ''' - - tickets_id = self.request('ticket.query', trac_query) - self.custom_fields = self.get_custom_fields() - - for ticket_id in tickets_id: - pagure_issue = self.create_issue(ticket_id) - pagure_issue.comments = [] - pagure_issue_comments = self.request('ticket.changeLog', ticket_id) - comments = self.create_comments(pagure_issue_comments) - # add all the comments to the issue object - for key in comments: - if comments[key].attachment is not None and \ - any(attachment in comments[key].attachment for attachment in - pagure_issue.attachment): - - for attach_name in comments[key].attachment: - filename = get_secure_filename( - pagure_issue.attachment[attach_name], attach_name) - url = '/%s/issue/raw/files/%s' % (repo_name, filename) - if is_image(attach_name): - comments[key].comment += ('\n[![%s](%s)](%s)' % - (attach_name, url, url)) - else: - comments[key].comment += ('\n[%s](%s)' % - (attach_name, url)) - pagure_issue.comments.append(comments[key].to_json()) - click.echo('Updated ' + repo_name + ' with issue :' + - str(ticket_id) + '/' + str(tickets_id[-1])) - issue_to_json(pagure_issue, self.clone_repo_location) - - def get_custom_fields_of_ticket(self, trac_ticket): - ''' Given the trac ticket, it will return all the - custom fields of the ticket, in a form that it can - be used for pagure Issue ''' - - pagure_fields = [] - for field in self.custom_fields: - if field['name'] in trac_ticket: - pagure_field = {} - pagure_field['name'] = field.get('name') - pagure_field['key_type'] = field.get('key_type') - pagure_field['value'] = trac_ticket.get( - pagure_field['name'], "").strip() - if pagure_field['value']: - pagure_fields.append(pagure_field) - return pagure_fields - - def create_issue(self, ticket_id): - ''' Create Issue object from track ticket ''' - - trac_ticket_info = self.request('ticket.get', ticket_id) - trac_ticket = trac_ticket_info[3] - trac_attachments = self.request('ticket.listAttachments', ticket_id) - - pagure_attachment = {} - for attachment in trac_attachments: - filename = attachment[0] - attachment_resp = self.request( - 'ticket.getAttachment', - ticket_id, filename) - if attachment_resp: - if is_image(filename): - content = b64decode(attachment_resp['__jsonclass__'][1]) - pagure_attachment[filename] = content - else: - content = b64decode( - attachment_resp['__jsonclass__'][1].replace('\n', '')) - pagure_attachment[filename] = content - - pagure_custom_fields = self.get_custom_fields_of_ticket(trac_ticket) - pagure_issue_title = trac_ticket['summary'] - - pagure_issue_content = trac_ticket['description'] - if pagure_issue_content == '': - pagure_issue_content = '#No Description Provided' - - issue_status, close_status = self.get_ticket_status(trac_ticket) - - pagure_issue_created_at = to_timestamp( - trac_ticket_info[1]['__jsonclass__'][1]) - - if self.fas: - pagure_issue_assignee = self.fas.find_fas_user( - trac_ticket['owner']) - pagure_issue_user = self.fas.find_fas_user(trac_ticket['reporter']) - if not pagure_issue_user.name: - pagure_issue_user = User( - name=trac_ticket['reporter'], - fullname=trac_ticket['reporter'], - emails=[trac_ticket['reporter'] + '@fedoraproject.org']) - else: - pagure_issue_assignee = User(name='', fullname='', emails=[]) - pagure_issue_user = User( - name=trac_ticket['reporter'], - fullname=trac_ticket['reporter'], - emails=[trac_ticket['reporter'] + '@fedoraproject.org']) - - # The milestone of the issue - pagure_milestone = None - if 'milestone' in trac_ticket and trac_ticket['milestone'] != '': - pagure_milestone = trac_ticket['milestone'] - - # Issue tags - pagure_issue_tags = [] - if self.tags: - pagure_issue_tags = filter( - lambda x: x != '', trac_ticket['keywords'].split(' ')) - - pagure_issue_tags = self.pre_process_tags(pagure_issue_tags) - - pagure_issue_depends = [] - pagure_issue_blocks = [] - if self.private: - pagure_issue_is_private = True - else: - pagure_issue_is_private = False - - pagure_issue = Issue( - id=ticket_id + self.offset, - title=pagure_issue_title, - content=pagure_issue_content, - status=issue_status, - close_status=close_status, - date_created=pagure_issue_created_at, - user=pagure_issue_user.to_json(), - private=pagure_issue_is_private, - attachment=pagure_attachment, - tags=pagure_issue_tags, - milestone=pagure_milestone, - depends=pagure_issue_depends, - blocks=pagure_issue_blocks, - assignee=pagure_issue_assignee.to_json(), - custom_fields=pagure_custom_fields,) - return pagure_issue - - def pre_process_tags(self, tags): - ''' Pre process the tags before sending it to pagure ''' - - delims_str = ',|/|;|:|-|\+|\*|\'|\"' - - # Remove the delims between the tags and convert all tags to lower case - pagure_issue_tags = list(set([j for k in [ - re.split(delims_str, i.lower()) for i in tags] for j in k])) - - return pagure_issue_tags - - def get_ticket_status(self, trac_ticket): - ''' Returns the corresponding status of ticket on pagure ''' - close_status = get_close_status() - if close_status is not None: - if trac_ticket['status'] != 'closed': - return ('Open', '') - else: - for status in close_status: - if trac_ticket['resolution'] in close_status[status]: - return ('Closed', status) - return ('Closed', 'Fixed') - else: - click.echo('ERROR: Close Status not read from config file') - sys.exit(1) +def pre_process_tags(tags): + ''' Pre process the tags before sending it to pagure ''' - def get_comment_user(self, comment): - ''' Returns the user who commented on the ticket ''' + delims_str = ',|/|;|:|-|\+|\*|\'|\"' - # The User who commented - if self.fas and comment[1]: - pagure_issue_comment_user = self.fas.find_fas_user(comment[1]) - if not pagure_issue_comment_user.name: - pagure_issue_comment_user = self.somebody - else: - pagure_issue_comment_user = self.somebody - return pagure_issue_comment_user + # Remove the delims between the tags and convert all tags to lower case + pagure_issue_tags = list(set([j for k in [ + re.split(delims_str, i.lower()) for i in tags] for j in k])) - def create_comments(self, trac_comments): - ''' Create IssueComment objects from the trac comments ''' + return pagure_issue_tags - comments = {} - for comment in trac_comments: - ts = to_timestamp(comment[0]['__jsonclass__'][1]) - if comment[2] == 'comment' and comment[4] != '': - if ts in comments: - attachment = comments[ts].attachment - else: - attachment = [] +def request(method, rest, *args): + ''' Common method for querying trac ''' - pagure_issue_comment_body = comment[4] - pagure_issue_comment_created_at = ts + reqid = randint(0, 100) + req = {'params': args, + 'method': method, + 'id': reqid} + resp = requests.post(rest.url, json=req, + auth=(rest.username, rest.password)) + resp = resp.json() + if resp['id'] != reqid: + click.echo('ERROR: Invalid response for request! ' + + 'ID does not match') + sys.exit(1) + if resp['error'] is not None: + # Ignore missing attachment errors + if 'Attachment ' not in resp['error']['message'] and \ + ' not found' not in resp['error']['message']: + click.echo("ERROR: Error in response: %s" % resp['error']) + sys.exit(1) + return resp['result'] + + +def import_issues(repo_name, rest, trac_query='max=0&order=id'): + ''' Queries the trac instance via its jsonrpc API and convert the + tickets into JSON blob to be imported into pagure's ticket git repo. + + :arg repo_name: the name of the repository + :arg repo_folder: the folder in which is the repository + :kwarg trac_query: the query to call trac with in order to retrieve + all the tickets. + Defaults to ``max=0&order=id`` + + ''' + + tickets_id = request('ticket.query', rest, trac_query) + ticket_fields = request('ticket.getTicketFields', rest) + custom_fields = get_custom_fields(ticket_fields) + clone_repo_location = os.path.join(REPO_PATH, 'clone-' + repo_name) + + for ticket_id in tickets_id: + pagure_issue = create_issue(ticket_id=ticket_id, + custom_fields=custom_fields, + rest=rest) + pagure_issue.comments = [] + pagure_issue_comments = request('ticket.changeLog', rest, ticket_id) + comments = create_comments(pagure_issue_comments) + # add all the comments to the issue object + pagure_issue = add_comment_to_issue(comments, pagure_issue) + + click.echo('Updated ' + repo_name + ' with issue :' + + str(ticket_id) + '/' + str(tickets_id[-1])) + issue_to_json(pagure_issue, clone_repo_location) + + +def get_custom_fields(ticket_fields): + ''' Queries the fedorahosted api to get all ticket fields + and filters all the custom fields, returns + a list of dicts - dict with keys 'name' and 'key_type' ''' + + custom_fields = [] + for field in ticket_fields: + if field.get('custom') is True: + current_field = {} + current_field['name'] = field['name'] + key_type = 'text' + if field['type'] == 'checkbox': + key_type = 'boolean' + current_field['key_type'] = key_type + custom_fields.append(current_field) + return custom_fields + + +def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags): + ''' Create Issue object from track ticket ''' + + trac_ticket_info = request('ticket.get', rest, ticket_id) + trac_ticket = trac_ticket_info[3] + trac_attachments = request('ticket.listAttachments', rest, ticket_id) + + pagure_attachment = {} + for attachment in trac_attachments: + filename = attachment[0] + attachment_resp = request('ticket.getAttachment', + rest, ticket_id, filename) + if attachment_resp: + if is_image(filename): + content = b64decode(attachment_resp['__jsonclass__'][1]) + pagure_attachment[filename] = content + else: + content = b64decode( + attachment_resp['__jsonclass__'][1].replace('\n', '')) + pagure_attachment[filename] = content + + pagure_custom_fields = get_custom_fields_of_ticket(trac_ticket=trac_ticket, + custom_fields=custom_fields) + pagure_issue_title = trac_ticket['summary'] + + pagure_issue_content = trac_ticket['description'] + if pagure_issue_content == '': + pagure_issue_content = '#No Description Provided' + + issue_status, close_status = get_ticket_status(trac_ticket) + + pagure_issue_created_at = to_timestamp( + trac_ticket_info[1]['__jsonclass__'][1]) + + fas = FASclient(rest.username, rest.password, + 'https://admin.fedoraproject.org/accounts') + pagure_issue_assignee = fas.find_fas_user(trac_ticket['owner']) + pagure_issue_user = fas.find_fas_user(trac_ticket['reporter']) + if not pagure_issue_user.name: + pagure_issue_user = User( + name=trac_ticket['reporter'], + fullname=trac_ticket['reporter'], + emails=[trac_ticket['reporter'] + '@fedoraproject.org']) + + # The milestone of the issue + pagure_milestone = None + if 'milestone' in trac_ticket and trac_ticket['milestone'] != '': + pagure_milestone = trac_ticket['milestone'] + + # Issue tags + pagure_issue_tags = [] + if tags: + pagure_issue_tags = filter( + lambda x: x != '', trac_ticket['keywords'].split(' ')) + + pagure_issue_tags = pre_process_tags(pagure_issue_tags) + + pagure_issue_depends = [] + pagure_issue_blocks = [] + if private: + pagure_issue_is_private = True + else: + pagure_issue_is_private = False + + pagure_issue = Issue( + id=ticket_id + offset, + title=pagure_issue_title, + content=pagure_issue_content, + status=issue_status, + close_status=close_status, + date_created=pagure_issue_created_at, + user=pagure_issue_user.to_json(), + private=pagure_issue_is_private, + attachment=pagure_attachment, + tags=pagure_issue_tags, + milestone=pagure_milestone, + depends=pagure_issue_depends, + blocks=pagure_issue_blocks, + assignee=pagure_issue_assignee.to_json(), + custom_fields=pagure_custom_fields,) + return pagure_issue + + +def get_custom_fields_of_ticket(trac_ticket, custom_fields): + ''' Given the trac ticket, it will return all the + custom fields of the ticket, in a form that it can + be used for pagure Issue ''' + + pagure_fields = [] + for field in custom_fields: + if field['name'] in trac_ticket: + pagure_field = {} + pagure_field['name'] = field.get('name') + pagure_field['key_type'] = field.get('key_type') + pagure_field['value'] = trac_ticket.get(pagure_field['name']) + pagure_fields.append(pagure_field) + return pagure_fields + + +def get_ticket_status(trac_ticket): + ''' Returns the corresponding status of ticket on pagure ''' + close_status = get_close_status() + if close_status is not None: + if trac_ticket['status'] != 'closed': + return ('Open', '') + else: + for status in close_status: + if trac_ticket['resolution'] in close_status[status]: + return ('Closed', status) + return ('Closed', 'Fixed') + else: + click.echo('ERROR: Close Status not read from config file') + sys.exit(1) + + +def create_comments(trac_comments): + ''' Create IssueComment objects from the trac comments ''' + + comments = {} + for comment in trac_comments: + ts = to_timestamp(comment[0]['__jsonclass__'][1]) + + if comment[2] == 'comment' and comment[4] != '': + if ts in comments: + attachment = comments[ts].attachment + else: + attachment = [] + + pagure_issue_comment_body = comment[4] + pagure_issue_comment_created_at = ts + + pagure_issue_comment_user = get_comment_user(comment) + # Object to represent comment on an issue + comments[ts] = IssueComment( + id=None, + comment=pagure_issue_comment_body, + date_created=pagure_issue_comment_created_at, + attachment=attachment, + user=pagure_issue_comment_user.to_json()) + + elif comment[2] == 'attachment': + if ts in comments: + comments[ts].attachment.append(comment[4]) + else: pagure_issue_comment_user = self.get_comment_user(comment) - # Object to represent comment on an issue comments[ts] = IssueComment( id=None, - comment=pagure_issue_comment_body, - date_created=pagure_issue_comment_created_at, - attachment=attachment, + comment='attachment', + date_created=ts, + attachment=[comment[4]], user=pagure_issue_comment_user.to_json()) - elif comment[2] == 'attachment': - if ts in comments: - comments[ts].attachment.append(comment[4]) + return comments + + +def get_comment_user(comment, fas): + ''' Returns the user who commented on the ticket ''' + + # The User who commented + if fas and comment[1]: + pagure_issue_comment_user = fas.find_fas_user(comment[1]) + if not pagure_issue_comment_user.name: + pagure_issue_comment_user = SOMEBODY + else: + pagure_issue_comment_user = SOMEBODY + return pagure_issue_comment_user + + +def add_comment_to_issue(comments, pagure_issue, repo_name): + '''Adding comment to an issue''' + for key in comments: + if comments[key].attachment is not None and \ + any(attachment in comments[key].attachment for attachment in + pagure_issue.attachment): + + for attach_name in comments[key].attachment: + filename = get_secure_filename( + pagure_issue.attachment[attach_name], attach_name) + url = '/%s/issue/raw/files/%s' % (repo_name, filename) + if is_image(attach_name): + comments[key].comment += ('\n[![%s](%s)](%s)' % + (attach_name, url, url)) else: - pagure_issue_comment_user = self.get_comment_user(comment) - comments[ts] = IssueComment( - id=None, - comment='attachment', - date_created=ts, - attachment=[comment[4]], - user=pagure_issue_comment_user.to_json()) - - return comments + comments[key].comment += ('\n[%s](%s)' % + (attach_name, url)) + pagure_issue.comments.append(comments[key].to_json()) + return pagure_issue From cb8e1f576cee7efd168f0c24a65b9962258727f7 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Feb 12 2017 11:05:54 +0000 Subject: [PATCH 6/8] Move logic to import issues into fedorahosted command code Signed-off-by: Clement Verna --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 0f3004d..cc4d822 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -1,9 +1,13 @@ +import os +from collections import namedtuple import click import pagure_importer from pagure_importer.app import app, REPO_PATH -from pagure_importer.utils import importer_trac, get_pagure_namespace +from pagure_importer.utils import get_pagure_namespace, issue_to_json +from pagure_importer.utils.fas import FASclient import pagure_importer.utils.git as gitutils +import pagure_importer.utils.importer_trac as libtrac @app.command() @@ -19,36 +23,60 @@ import pagure_importer.utils.git as gitutils help='Number of issue in pagure before import') @click.option('--nopush', is_flag=True, help="Do not push the result of pagure-importer back") -def fedorahosted( - project_url, tags, private, username, password, offset, nopush): - fasclient = FASclient(username, password, - 'https://admin.fedoraproject.org/accounts') +def fedorahosted(project_url, tags, private, username, password, + offset, nopush): + """Import issues from fedorahosted""" + if project_url.endswith('.git'): project_url = project_url.replace('.git', '') project_url += '/login/jsonrpc' repos = pagure_importer.utils.display_repo() + if repos: repo_index = click.prompt('Choose the import destination repo ', default=1) repo_name = repos[int(repo_index)-1] - newpath, new_repo = gitutils.clone_repo(repo_name, REPO_PATH) + _, new_repo = gitutils.clone_repo(repo_name, REPO_PATH) project = get_pagure_namespace(REPO_PATH, repo_name) - with importer_trac.TracImporter(project_url=project_url, - username=username, - password=password, - offset=offset, - repo_name=repo_name, - repo_folder=REPO_PATH, - fasclient=fasclient, - tags=tags, - private=private, - nopush=nopush) as trac_importer: - - trac_importer.import_issues(project) + rest_param = namedtuple('rest_param', 'url username password') + rest = rest_param(project_url, username, password) + fas = FASclient(rest.username, rest.password, + 'https://admin.fedoraproject.org/accounts') + + # import issues + tickets_id = libtrac.get_project_tickets(rest=rest) + ticket_fields = libtrac.get_project_tickets_fields(rest=rest) + custom_fields = libtrac.get_custom_fields(ticket_fields=ticket_fields) + clone_repo_location = os.path.join(REPO_PATH, 'clone-' + repo_name) + + for ticket_id in tickets_id: + + pagure_issue = libtrac.create_issue(ticket_id=ticket_id, + custom_fields=custom_fields, + rest=rest, + fas=fas, + offset=offset, + private=private, + tags=tags) + pagure_issue.comments = [] + pagure_issue_comments = libtrac.get_ticket_comments(rest=rest, + ticket_id=ticket_id) + comments = libtrac.create_comments(trac_comments=pagure_issue_comments, + fas=fas) + # add all the comments to the issue object + pagure_issue = libtrac.add_comment_to_issue(comments=comments, + pagure_issue=pagure_issue, + repo_name=project) + + click.echo('Updated ' + repo_name + ' with issue :' + + str(ticket_id) + '/' + str(tickets_id[-1])) + issue_to_json(issue=pagure_issue, folder=clone_repo_location) + # update the local git repo new_repo = gitutils.update_git( new_repo, commit_message='Imported issues from fedorahosted project: %s' % repo_name) + if not nopush: gitutils.push_repo(new_repo) else: From ce3531f9fb16f50f88160a397766770d46da5bc4 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Feb 12 2017 11:05:54 +0000 Subject: [PATCH 7/8] Remove TracImporter object and use function instead Signed-off-by: Clement Verna --- diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index 004a7bd..0457883 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -1,17 +1,14 @@ -import os import sys import re import time -import click -import requests from random import randint from base64 import b64decode from datetime import datetime +import click +import requests from pagure_importer.utils import ( - get_close_status, is_image, issue_to_json, get_secure_filename) + get_close_status, is_image, get_secure_filename) from pagure_importer.utils.models import User, Issue, IssueComment -from pagure_importer.utils.fas import FASclient -from pagure_importer.app import REPO_PATH SOMEBODY = User(name='somebody', fullname='somebody', emails=['some@body.com']) @@ -29,7 +26,7 @@ def to_timestamp(tm): def pre_process_tags(tags): ''' Pre process the tags before sending it to pagure ''' - delims_str = ',|/|;|:|-|\+|\*|\'|\"' + delims_str = r',|/|;|:|-|\+|\*|\'|\"' # Remove the delims between the tags and convert all tags to lower case pagure_issue_tags = list(set([j for k in [ @@ -62,36 +59,33 @@ def request(method, rest, *args): return resp['result'] -def import_issues(repo_name, rest, trac_query='max=0&order=id'): - ''' Queries the trac instance via its jsonrpc API and convert the - tickets into JSON blob to be imported into pagure's ticket git repo. - - :arg repo_name: the name of the repository - :arg repo_folder: the folder in which is the repository - :kwarg trac_query: the query to call trac with in order to retrieve - all the tickets. - Defaults to ``max=0&order=id`` +def get_project_tickets(rest, trac_query='max=0&order=id'): + """ Queries the trac instance via the jsonrpc API to get a list + tickets. By default returns all the project tickets + :arg rest: the rest connection parameters (url, username, password) + :arg trac_query: query use by trac to return tickets + """ + ticket_id = request('ticket.query', rest, trac_query) + return ticket_id - ''' - tickets_id = request('ticket.query', rest, trac_query) +def get_project_tickets_fields(rest): + """ Queries the trac instance via the jsonrpc API to get a list + of all the field used in the trac tickets + :arg rest: the rest connection parameters (url, username, password) + """ ticket_fields = request('ticket.getTicketFields', rest) - custom_fields = get_custom_fields(ticket_fields) - clone_repo_location = os.path.join(REPO_PATH, 'clone-' + repo_name) + return ticket_fields - for ticket_id in tickets_id: - pagure_issue = create_issue(ticket_id=ticket_id, - custom_fields=custom_fields, - rest=rest) - pagure_issue.comments = [] - pagure_issue_comments = request('ticket.changeLog', rest, ticket_id) - comments = create_comments(pagure_issue_comments) - # add all the comments to the issue object - pagure_issue = add_comment_to_issue(comments, pagure_issue) - click.echo('Updated ' + repo_name + ' with issue :' + - str(ticket_id) + '/' + str(tickets_id[-1])) - issue_to_json(pagure_issue, clone_repo_location) +def get_ticket_comments(rest, ticket_id): + """ Queries the trac instance via the jsonrpc API to get a list + of all the ticket's comments + :arg rest: the rest connection parameters (url, username, password) + :arg ticket_id: id of the ticket to query + """ + issue_comments = request('ticket.changeLog', rest, ticket_id) + return issue_comments def get_custom_fields(ticket_fields): @@ -112,18 +106,51 @@ def get_custom_fields(ticket_fields): return custom_fields +def get_ticket_info(rest, ticket_id): + """ Queries the trac instance via the jsonrpc API to get the + details of a ticket. + :arg rest: the rest connection parameters (url, username, password) + :arg ticket_id: id of the ticket to query + """ + ticket_info = request('ticket.get', rest, ticket_id) + return ticket_info + + +def get_ticket_attachments(rest, ticket_id): + """ Queries the trac instance via the jsonrpc API to get all + of a ticket's attachments. + :arg rest: the rest connection parameters (url, username, password) + :arg ticket_id: id of the ticket to query + """ + attachments = request('ticket.listAttachments', rest, ticket_id) + return attachments + + +def get_attachment(rest, ticket_id, attachment): + """ Queries the trac instance via the jsonrpc API to get a + ticket's attachment. + :arg rest: the rest connection parameters (url, username, password) + :arg ticket_id: id of the ticket to query + :arg attachment: name of the attachment + """ + attachment = request('ticket.getAttachment', + rest, ticket_id, attachment) + return attachment + + def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags): ''' Create Issue object from track ticket ''' - trac_ticket_info = request('ticket.get', rest, ticket_id) + trac_ticket_info = get_ticket_info(rest=rest, ticket_id=ticket_id) trac_ticket = trac_ticket_info[3] - trac_attachments = request('ticket.listAttachments', rest, ticket_id) + trac_attachments = get_ticket_attachments(rest=rest, ticket_id=ticket_id) pagure_attachment = {} for attachment in trac_attachments: filename = attachment[0] - attachment_resp = request('ticket.getAttachment', - rest, ticket_id, filename) + attachment_resp = get_attachment(rest=rest, + ticket_id=ticket_id, + attachment=filename) if attachment_resp: if is_image(filename): content = b64decode(attachment_resp['__jsonclass__'][1]) @@ -146,8 +173,6 @@ def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags): pagure_issue_created_at = to_timestamp( trac_ticket_info[1]['__jsonclass__'][1]) - fas = FASclient(rest.username, rest.password, - 'https://admin.fedoraproject.org/accounts') pagure_issue_assignee = fas.find_fas_user(trac_ticket['owner']) pagure_issue_user = fas.find_fas_user(trac_ticket['reporter']) if not pagure_issue_user.name: @@ -171,10 +196,6 @@ def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags): pagure_issue_depends = [] pagure_issue_blocks = [] - if private: - pagure_issue_is_private = True - else: - pagure_issue_is_private = False pagure_issue = Issue( id=ticket_id + offset, @@ -184,7 +205,7 @@ def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags): close_status=close_status, date_created=pagure_issue_created_at, user=pagure_issue_user.to_json(), - private=pagure_issue_is_private, + private=private, attachment=pagure_attachment, tags=pagure_issue_tags, milestone=pagure_milestone, @@ -227,7 +248,7 @@ def get_ticket_status(trac_ticket): sys.exit(1) -def create_comments(trac_comments): +def create_comments(trac_comments, fas): ''' Create IssueComment objects from the trac comments ''' comments = {} @@ -243,7 +264,7 @@ def create_comments(trac_comments): pagure_issue_comment_body = comment[4] pagure_issue_comment_created_at = ts - pagure_issue_comment_user = get_comment_user(comment) + pagure_issue_comment_user = get_comment_user(comment, fas) # Object to represent comment on an issue comments[ts] = IssueComment( id=None, @@ -256,7 +277,7 @@ def create_comments(trac_comments): if ts in comments: comments[ts].attachment.append(comment[4]) else: - pagure_issue_comment_user = self.get_comment_user(comment) + pagure_issue_comment_user = get_comment_user(comment, fas) comments[ts] = IssueComment( id=None, comment='attachment', From be87612049548e9395e98691bbcc81a7c24db619 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Feb 12 2017 11:05:54 +0000 Subject: [PATCH 8/8] Remove TracImporter object from test suite Signed-off-by: Clement Verna --- diff --git a/tests/test_pgimport_importer_trac.py b/tests/test_pgimport_importer_trac.py index fe3886b..a2c0539 100644 --- a/tests/test_pgimport_importer_trac.py +++ b/tests/test_pgimport_importer_trac.py @@ -1,28 +1,23 @@ import unittest from unittest.mock import MagicMock -from pagure_importer.utils.importer_trac import (TracImporter, to_timestamp) +from pagure_importer.utils.fas import FASclient +from pagure_importer.utils.models import User +import pagure_importer.utils.importer_trac as libtrac class PgimportImporterTrac (unittest.TestCase): def setUp(self): - self.trac = TracImporter(project_url="https://foobar.bar", - username="foo", - password="bar", - offset=0, - repo_folder="myfolder", - repo_name="myname", - nopush=False) + self.fas = FASclient('foo', 'bar', 'http://example.com') + self.fas_mock = MagicMock() + self.fas.find_fas_user = self.fas_mock + self.fas.find_fas_user.return_value = User(name='', + fullname='', + emails=['']) + self.somebody = {'name': 'somebody', 'fullname': 'somebody', 'emails': ['some@body.com']} - self.mock_request = MagicMock() - self.old_request = TracImporter.request - TracImporter.request = self.mock_request - - def tearDown(self): - TracImporter.request = self.old_request - def test_import_2_comments(self): # Case 1 - Import 2 comments @@ -32,9 +27,9 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2016-11-08T20:32:20']}, u'foobar', u'comment', u'2', u'Well this will improve the code base :).', 1]] - results = TracImporter.create_comments(self.trac, trac_comments) + results = libtrac.create_comments(trac_comments, self.fas) - ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) + ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) self.assertEqual("Yeah, let\'s have some test.", results[ts_1].comment) @@ -42,7 +37,7 @@ class PgimportImporterTrac (unittest.TestCase): self.assertEqual([], results[ts_1].attachment) self.assertEqual(self.somebody, results[ts_1].user) - ts_2 = to_timestamp(trac_comments[1][0]['__jsonclass__'][1]) + ts_2 = libtrac.to_timestamp(trac_comments[1][0]['__jsonclass__'][1]) self.assertEqual("Well this will improve the code base :).", results[ts_2].comment) @@ -56,7 +51,7 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2016-11-08T20:30:13']}, u'foobar', u'comment', u'1', u'', 1]] - results = TracImporter.create_comments(self.trac, trac_comments) + results = libtrac.create_comments(trac_comments, self.fas) self.assertEqual({}, results) def test_import_not_a_comments(self): @@ -65,7 +60,7 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2014-04-09T16:02:20']}, u'foobar', u'resolution', u'', u'wontfix', 1]] - results = TracImporter.create_comments(self.trac, trac_comments) + results = libtrac.create_comments(trac_comments, self.fas) self.assertEqual({}, results) def test_import_attachment(self): @@ -74,8 +69,8 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2014-10-31T12:32:18']}, u'foobar', u'attachment', u'', u'mytest.png', 0]] - results = TracImporter.create_comments(self.trac, trac_comments) - ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) + results = libtrac.create_comments(trac_comments, self.fas) + ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) self.assertEqual("attachment", results[ts_1].comment) @@ -91,8 +86,8 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2014-10-31T12:32:18']}, u'foobar', u'comment', u'', u'test attachment', 0]] - results = TracImporter.create_comments(self.trac, trac_comments) - ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) + results = libtrac.create_comments(trac_comments, self.fas) + ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) self.assertEqual("test attachment", results[ts_1].comment) @@ -108,8 +103,8 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2014-10-31T12:32:18']}, u'foobar', u'attachment', u'', u'mytest.png', 0]] - results = TracImporter.create_comments(self.trac, trac_comments) - ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) + results = libtrac.create_comments(trac_comments, self.fas) + ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) self.assertEqual("test attachment", results[ts_1].comment) @@ -127,8 +122,8 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2014-10-31T12:32:18']}, u'foobar', u'comment', u'', u'test 2 attachments', 0]] - results = TracImporter.create_comments(self.trac, trac_comments) - ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) + results = libtrac.create_comments(trac_comments, self.fas) + ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) self.assertEqual("test 2 attachments", results[ts_1].comment) @@ -140,60 +135,67 @@ class PgimportImporterTrac (unittest.TestCase): trac_ticket = {'status': 'open'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Open', ''), results) def test_get_ticket_status_closedInvalid(self): trac_ticket = {'status': 'closed', 'resolution': 'invalid'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Closed', 'Invalid'), results) def test_get_ticket_status_closedWontFix(self): trac_ticket = {'status': 'closed', 'resolution': 'wontfix'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Closed', 'Invalid'), results) def test_get_ticket_status_closedWorksforme(self): trac_ticket = {'status': 'closed', 'resolution': 'worksforme'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Closed', 'Invalid'), results) def test_get_ticket_status_closedDuplicate(self): trac_ticket = {'status': 'closed', 'resolution': 'duplicate'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Closed', 'Duplicate'), results) def test_get_ticket_status_closedInsufficient(self): trac_ticket = {'status': 'closed', 'resolution': 'insufficient_info'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Closed', 'Insufficient data'), results) def test_get_ticket_status_closedFixed(self): trac_ticket = {'status': 'closed', 'resolution': 'fixed'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Closed', 'Fixed'), results) def test_get_custom_fieldsText(self): - self.mock_request.return_value = [{'custom': True, 'name': 'foo', 'type': 'textarea'}] - results = TracImporter.get_custom_fields(self.trac) + ticket_fields = [{'custom': True, 'name': 'foo', 'type': 'textarea'}] + results = libtrac.get_custom_fields(ticket_fields) self.assertEqual([{'key_type': 'text', 'name': 'foo'}], results) def test_get_custom_fieldsBoolean(self): - self.mock_request.return_value = [{'custom': True, 'name': 'bar', 'type': 'checkbox'}] - results = TracImporter.get_custom_fields(self.trac) + ticket_fields = [{'custom': True, 'name': 'bar', 'type': 'checkbox'}] + results = libtrac.get_custom_fields(ticket_fields) self.assertEqual([{'key_type': 'boolean', 'name': 'bar'}], results) def test_get_tickets_custom_fiedls(self): - self.trac.custom_fields = [{'key_type': 'text', 'name': 'bar'}, - {'key_type': 'boolean', 'name': 'foo'}] + custom_fields = [{'key_type': 'text', 'name': 'bar'}, + {'key_type': 'boolean', 'name': 'foo'}] trac_ticket = {'bar': 'hello', 'foo': 'world', 'foobar': 'hello world'} - results = TracImporter.get_custom_fields_of_ticket(self.trac, trac_ticket) - self.assertDictEqual({'key_type': 'text', 'value': 'hello', 'name': 'bar'}, results[0]) - self.assertDictEqual({'key_type': 'boolean', 'value': 'world', 'name': 'foo'}, results[1]) + results = libtrac.get_custom_fields_of_ticket(trac_ticket, custom_fields) + self.assertDictEqual({'key_type': 'text', 'value': 'hello', 'name': 'bar'}, + results[0]) + self.assertDictEqual({'key_type': 'boolean', 'value': 'world', 'name': 'foo'}, + results[1]) + + +if __name__ == '__main__': + SUITE = unittest.TestLoader().loadTestsFromTestCase(PgimportImporterTrac) + unittest.TextTestRunner(verbosity=2).run(SUITE)