From c6f38e1cb86c3abf3692e06b990efc9bdf20dcbc Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Feb 23 2016 08:22:42 +0000 Subject: [PATCH 1/24] Model to represent an issue as json --- diff --git a/pagure_importer/lib/models.py b/pagure_importer/lib/models.py new file mode 100644 index 0000000..7ec0231 --- /dev/null +++ b/pagure_importer/lib/models.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +import datetime +import json +import uuid + +class Issue(): + ''' Represents an Issue ''' + + def __init__( + self, id, title, content, + status, date_created, user, private, tags, + depends, blocks, assignee, comments=None): + + self.id = id + self.title = title + self.content = content + self.status = status + self.date_created = date_created + self.user = user + self.private = private + self.tags = tags + self.depends = depends + self.blocks = blocks + self.assignee = assignee + self.comments = comments + self.uid = uuid.uuid4().hex + + def to_json(self): + ''' Returns a dictionary representation of the issue. + + ''' + output = { + 'id': self.id, + 'title': self.title, + 'content': self.content, + 'status': self.status, + 'date_created': self.date_created.strftime('%s'), + 'user': self.user, + 'private': self.private, + 'tags': self.tags, + 'depends': self.depends, + 'blocks': self.blocks, + 'assignee': self.assignee, + 'comments': self.comments + } + + return output + + @property + def isa(self): + return 'issue' + + +class IssueComment(): + ''' Represent a comment for an issue ''' + + def __init__( + self, id, comment, date_created, + user, parent=None, edited_on=None, editor=None): + + self.id = id + self.comment = comment + self.parent = parent + self.date_created = date_created + self.user = user + self.edited_on = edited_on + self.editor = editor + + def to_json(self): + ''' Returns a dictionary representation of the issue. ''' + + output = { + 'id': self.id, + 'comment': self.comment, + 'parent': self.parent, + 'date_created': self.date_created.strftime('%s'), + 'user': self.user, + 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, + 'editor': self.editor or None + } + return output + + +class User(): + ''' Represents a User ''' + + def __init__( + self, name, emails, + fullname=None): + self.name = name + self.fullname = fullname + self.emails = emails + + def to_json(self): + ''' Return a representation of the User in a dictionary. ''' + + output = { + 'name': self.name, + 'fullname': self.fullname, + 'emails': self.emails + } + + return output From 07b97f010463d3043701a35894668d0d19043fa8 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Feb 23 2016 08:36:47 +0000 Subject: [PATCH 2/24] added repo.py from pagure --- diff --git a/pagure_importer/lib/repo.py b/pagure_importer/lib/repo.py new file mode 100644 index 0000000..f46b31a --- /dev/null +++ b/pagure_importer/lib/repo.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py + by pingou@pingoured.fr +''' + + +import pygit2 +import sys + + +def get_pygit2_version(): + ''' Return pygit2 version as a tuple of integers. + This is needed for correct version comparison. + ''' + return tuple([int(i) for i in pygit2.__version__.split('.')]) + + +class PagureRepo(pygit2.Repository): + """ An utility class allowing to go around pygit2's inability to be + stable. + + """ + + @staticmethod + def push(remote, refname): + """ Push the given reference to the specified remote. """ + pygit2_version = get_pygit2_version() + if pygit2_version >= (0, 22): + remote.push([refname]) + else: + remote.push(refname) + + def pull(self, remote_name='origin', branch='master', force=False): + ''' pull changes for the specified remote (defaults to origin). + + Code from MichaelBoselowitz at: + https://github.com/MichaelBoselowitz/pygit2-examples/blob/ + 68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58 + licensed under the MIT license. + ''' + + for remote in self.remotes: + if remote.name == remote_name: + remote.fetch() + remote_master_id = self.lookup_reference( + 'refs/remotes/origin/%s' % branch).target + + if force: + repo_branch = self.lookup_reference( + 'refs/heads/%s' % branch) + repo_branch.set_target(remote_master_id) + + merge_result, _ = self.merge_analysis(remote_master_id) + # Up to date, do nothing + if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: + return + # We can just fastforward + elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: + self.checkout_tree(self.get(remote_master_id)) + master_ref = self.lookup_reference( + 'refs/heads/%s' % branch) + master_ref.set_target(remote_master_id) + self.head.set_target(remote_master_id) + elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: + sys.exit('Pulling remote changes leads to a conflict') + else: + print 'Unexpected merge result: %s' % ( + pygit2.GIT_MERGE_ANALYSIS_NORMAL) + raise AssertionError('Unknown merge analysis result') From 4830732621f4053fca4d4fbb3f12e60da34f8206 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Feb 23 2016 08:40:04 +0000 Subject: [PATCH 3/24] added git.py from pagure code --- diff --git a/pagure_importer/lib/git.py b/pagure_importer/lib/git.py new file mode 100644 index 0000000..eff314f --- /dev/null +++ b/pagure_importer/lib/git.py @@ -0,0 +1,98 @@ +''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py + by pingou@pingoured.fr +''' + +import shutil +import os +import pygit2 +import tempfile +import json + +from repo import * + +def update_git(obj, repo_path, repofolder): + """ Update the given issue in its git. + This method forks the provided repo, add/edit the issue whose file name + is defined by the uid field of the issue and if there are additions/ + changes commit them and push them back to the original repo. + """ + + if not repofolder: + return + + # Get the fork + repopath = os.path.join(repofolder, repo_path) + + # Clone the repo into a temp folder + newpath = tempfile.mkdtemp(prefix='pagure-') + new_repo = pygit2.clone_repository(repopath, newpath) + + file_path = os.path.join(newpath, obj.uid) + + # Get the current index + index = new_repo.index + + # Are we adding files + added = False + if not os.path.exists(file_path): + added = True + + # Write down what changed + with open(file_path, 'w') as stream: + stream.write(json.dumps( + obj.to_json(), sort_keys=True, indent=4, + separators=(',', ': '))) + + # Retrieve the list of files that changed + diff = new_repo.diff() + files = [] + for p in diff: + if hasattr(p, 'new_file_path'): + files.append(p.new_file_path) + elif hasattr(p, 'delta'): + files.append(p.delta.new_file.path) + + # Add the changes to the index + if added: + index.add(obj.uid) + for filename in files: + index.add(filename) + + # If not change, return + if not files and not added: + shutil.rmtree(newpath) + return + + # See if there is a parent to this commit + parent = None + try: + parent = new_repo.head.get_object().oid + except pygit2.GitError: + pass + + parents = [] + if parent: + parents.append(parent) + + # Author/commiter will always be this one + author = pygit2.Signature(name='pagure', email='pagure') + + # Actually commit + new_repo.create_commit( + 'refs/heads/master', + author, + author, + 'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title), + new_repo.index.write_tree(), + parents) + index.write() + + # Push to origin + ori_remote = new_repo.remotes[0] + master_ref = new_repo.lookup_reference('HEAD').resolve() + refname = '%s:%s' % (master_ref.name, master_ref.name) + + PagureRepo.push(ori_remote, refname) + + # Remove the clone + shutil.rmtree(newpath) From 4f74da44a6ca18c76fe74ac85fc7f3104421775f Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Apr 10 2016 10:54:24 +0000 Subject: [PATCH 4/24] Basics of pygit2 approach for github issues completed --- diff --git a/pagure_importer/__init__.py b/pagure_importer/__init__.py index de741ae..fbab797 100644 --- a/pagure_importer/__init__.py +++ b/pagure_importer/__init__.py @@ -1 +1,2 @@ -from sources import * +import lib +import settings diff --git a/pagure_importer/forms.py b/pagure_importer/forms.py index 1aa883a..ddc24f0 100644 --- a/pagure_importer/forms.py +++ b/pagure_importer/forms.py @@ -1,35 +1,10 @@ -from sources.importer_github import GithubImporter import getpass +from lib.sources.importer_github_new import GithubImporter +from settings import REPO_PATH, REPO_NAME + def form_github_issues(): github_username = raw_input('Enter you Github Username: ') github_password = getpass.getpass('Enter your github password: ') - github_project_name = raw_input('Enter github project name: ') - pagure_api_key = raw_input('Enter your pagure api key: ') - pagure_project_name = raw_input('Enter pagure project name: ') - - is_forked = raw_input('Is the pagure project a forked repo ? (y/n): ') or 'n' - if is_forked.lower() == 'y': - pagure_username = raw_input('Enter your pagure username: ') - else: - pagure_username = None - - is_pagure_io = raw_input( - 'Is the pagure instance url - https://pagure.io ?: (y/n) ') or 'y' - if is_pagure_io.lower() == 'n': - pagure_instance = raw_input('Enter the pagure instance url: ') or 'https://pagure.io' - else: - pagure_instance = 'https://pagure.io' - - status = raw_input( - 'Enter status of the issues to be imported (all/open/closed): ') or 'all' - - github_importer = GithubImporter( - github_username=github_username, - github_password=github_password, - github_project_name=github_project_name, - pagure_api_key=pagure_api_key, - pagure_project_name=pagure_project_name, - pagure_username=pagure_username, - instance_url=pagure_instance) - github_importer.import_issues(status) + github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') + return (github_username, github_password, github_project_name) diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py new file mode 100644 index 0000000..c1724e7 --- /dev/null +++ b/pagure_importer/lib/__init__.py @@ -0,0 +1,153 @@ +import git +import models + +import os +import getpass +import requests +import json +from github import Github +from requests.auth import HTTPBasicAuth +import pagure_importer +import pagure_importer.lib +from pagure_importer.lib.exceptions import FileNotFound, EmailNotFound + +def generate_json_for_github_contributors(github_username, github_password, \ + github_project_name): + ''' Creates a file containing a list of dicts containing the username and emails + of the contributors in the given github project + ''' + + github_obj = Github(github_username, github_password) + project = github_obj.get_repo(github_project_name) + commits_url = project.commits_url.replace('{/sha}', '') + + page = 0 + contributors = [] + while True: + page += 1 + payload = {'page': page } + data_ = json.loads(requests.get(commits_url, params=payload, auth=HTTPBasicAuth(github_username, github_password)).text) + + if not data_: + break + + for data in data_: + try: + contributor = data['commit']['committer'] + contributor_email = contributor['email'] + contributor_fullname = contributor['name'] + contributor_name = data['committer']['login'] + except TypeError: + print 'Maybe one of the contributors is dropped because of lack of details' + continue + + json_data = { + 'name': contributor_name, + 'fullname': contributor_fullname, + 'emails': [contributor_email] + } + + present = False + for i in contributors: + if i == json_data: + present = True + break + + if not present: + print 'contributor added: ', len(contributors) + 1 + contributors.append(json_data) + + with open('contributors.json', 'w') as f: + f.write(json.dumps(contributors)) + + return + + +def generate_json_for_github_issue_commentors(github_username, github_password, \ + github_project_name): + ''' Will create a json file containing details of all the user + who have commented on any issue in the given project + ''' + + github_obj = Github(github_username, github_password) + project = github_obj.get_repo(github_project_name) + issue_comment_url = project.issue_comment_url.replace('{/number}', '') + + page = 0 + issue_commentors = [] + while True: + page += 1 + payload = {'page': page } + data_ = json.loads(requests.get(issue_comment_url, params=payload, auth=HTTPBasicAuth(github_username, github_password)).text) + + if not data_: + break + + for data in data_: + try: + commentor = data['user']['login'] + except TypeError: + print 'Maybe one of the issue commentors have been dropped because of lack of details' + continue + + present = False + for i in issue_commentors: + if i == commentor: + present = True + break + + if not present: + print 'commentor added: ', len(issue_commentors) + 1 + issue_commentors.append(commentor) + + with open('issue_commentors.json', 'w') as f: + f.write(json.dumps(issue_commentors)) + return + + +def assemble_github_contributors_commentors(): + ''' It uses the files: issue_commentors.json and contributors.json + Assembles and creates a file: assembled_commentors.json + To use: just fill the None and [] in the final file ''' + + with open('issue_commentors.json', 'r') as ic: + issue_names = json.load(ic) + + with open('contributors.json', 'r') as c: + contributors = json.load(c) + + names = [] + for i in issue_names: + found = False + for j in contributors: + if j.get('name', None) == i: + names.append(j) + found = True + + if not found: + d = {'name': i, 'fullname': None, 'emails': []} + names.append(d) + + with open('assembled_commentors.json', 'w') as ac: + json.dump(names, ac) + + +def github_get_commentor_email(name): + ''' Will return the issue commentor email as given in the + assembled_commentors.json file + ''' + + if not os.path.exists('assembled_commentors.json'): + raise FileNotFound('The assembled_commentors.json file must be present \ + Rerun the program and choose to generate the json files') + + with open('assembled_commentors.json') as ac: + data = json.load(ac) + + for i in data: + if i.get('name', None) == name: + if i['emails']: + return i['emails'] + else: + raise EmailNotFound('You need to fill out all the emails of the \ + issue commentors') diff --git a/pagure_importer/lib/exceptions.py b/pagure_importer/lib/exceptions.py new file mode 100644 index 0000000..a5ab5fe --- /dev/null +++ b/pagure_importer/lib/exceptions.py @@ -0,0 +1,22 @@ +class GithubBadCredentials(Exception): + ''' Raised when username/password for github is wrong ''' + def __init__(self, msg): + self.msg = msg + + +class GithubRepoNotFound(Exception): + ''' Raised when the repo is not found for the user ''' + def __init__(self, msg): + self.msg = msg + + +class FileNotFound(Exception): + ''' Raised when a certain file is not found ''' + def __init__(self, msg): + self.msg = msg + + +class EmailNotFound(Exception): + ''' Raised when email is not found ''' + def __init__(self, msg): + self.msg = msg diff --git a/pagure_importer/lib/git.py b/pagure_importer/lib/git.py index eff314f..33eba48 100644 --- a/pagure_importer/lib/git.py +++ b/pagure_importer/lib/git.py @@ -10,18 +10,18 @@ import json from repo import * -def update_git(obj, repo_path, repofolder): +def update_git(obj, repo_path, repo_folder): """ Update the given issue in its git. This method forks the provided repo, add/edit the issue whose file name is defined by the uid field of the issue and if there are additions/ changes commit them and push them back to the original repo. """ - if not repofolder: + if not repo_folder: return # Get the fork - repopath = os.path.join(repofolder, repo_path) + repopath = os.path.join(repo_folder, repo_path) # Clone the repo into a temp folder newpath = tempfile.mkdtemp(prefix='pagure-') diff --git a/pagure_importer/lib/models.py b/pagure_importer/lib/models.py index 7ec0231..3aa3d83 100644 --- a/pagure_importer/lib/models.py +++ b/pagure_importer/lib/models.py @@ -79,6 +79,7 @@ class IssueComment(): 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, 'editor': self.editor or None } + return output diff --git a/pagure_importer/lib/sources/__init__.py b/pagure_importer/lib/sources/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pagure_importer/lib/sources/__init__.py diff --git a/pagure_importer/lib/sources/importer_github.py b/pagure_importer/lib/sources/importer_github.py new file mode 100644 index 0000000..2e635b6 --- /dev/null +++ b/pagure_importer/lib/sources/importer_github.py @@ -0,0 +1,133 @@ +from github import Github +import pagure_importer +import pagure_importer.lib +from pagure_importer.lib import models +from pagure_importer.lib import github_get_commentor_email +from pagure_importer.lib.exceptions import GithubBadCredentials, GithubRepoNotFound + +class GithubImporter(): + ''' Imports from Github using PyGithub and libpagure ''' + def __init__( + self, + github_username, + github_password, + github_project_name): + self.github_username = github_username + self.github_password = github_password + self.github_project_name = github_project_name + self.github = Github(github_username, github_password) + + def import_issues(self, repo_path, repo_folder, status='all'): + ''' Imports the issues on github for + the given project + ''' + github_user = None + try: + github_user = self.github.get_user(self.github_username) + except: + raise GithubBadCredentials( + 'Given github credentials are not correct') + repo = self.github.get_repo(self.github_project_name) + try: + repo_name = repo.name + except: + raise GithubRepoNotFound( + 'Repo not found, project name wrong') + + for github_issue in repo.get_issues(state=status): + + #title of the issue + pagure_issue_title = github_issue.title + + #body of the issue + if github_issue.body: + pagure_issue_content = github_issue.body + else: + pagure_issue_content = '#No Description Provided' + + #Some details of a issue + if github_issue.state != 'closed': + pagure_issue_status = 'Open' + else: + pagure_issue_status = 'Fixed' + + pagure_issue_created_at = github_issue.created_at + + #Not sure how to deal with this atm + pagure_issue_assignee = None + + if github_issue.labels: + pagure_issue_tags = [i.name for i in github_issue.labels] + else: + pagure_issue_tags = [] + + + #few things not supported by github + pagure_issue_depends = [] + pagure_issue_blocks = [] + pagure_issue_is_private = False + + + #User who created the issue + pagure_issue_user = models.User( + name=github_issue.user.login, + fullname=github_issue.user.name, + emails=[github_issue.user.email]) + + + pagure_issue = models.Issue( + id=None, + title = pagure_issue_title, + content = pagure_issue_content, + status = pagure_issue_status, + date_created = pagure_issue_created_at, + user = pagure_issue_user.to_json(), + private = pagure_issue_is_private, + tags = pagure_issue_tags, + depends = pagure_issue_depends, + blocks = pagure_issue_blocks, + assignee = pagure_issue_assignee) + + + #comments on the issue + comments = [] + for comment in github_issue.get_comments(): + + comment_user = comment.user + pagure_issue_comment_user_email = comment_user.email + pagure_issue_comment_body = comment.body + pagure_issue_comment_created_at = comment.created_at + pagure_issue_comment_updated_at = comment.updated_at + + + #No idea what to do with this right now + #editor: not supported by github api + pagure_issue_comment_parent = None + pagure_issue_comment_editor = None + + #comment updated at + pagure_issue_comment_edited_on = comment.updated_at + + #The User who commented + pagure_issue_comment_user = models.User( + name=comment_user.login, + fullname=comment_user.name, + emails=[comment_user.email] if comment_user.email else github_get_commentor_email(comment_user.login)) + + #Object to represent comment on an issue + pagure_issue_comment = models.IssueComment( + id=None, + comment=pagure_issue_comment_body, + parent=pagure_issue_comment_parent, + date_created=pagure_issue_comment_created_at, + user=pagure_issue_comment_user.to_json(), + edited_on=pagure_issue_comment_edited_on, + editor=pagure_issue_comment_editor) + + comments.append(pagure_issue_comment.to_json()) + + #add all the comments to the issue object + pagure_issue.comments = comments + + #update the local git repo + pagure_importer.lib.git.update_git(pagure_issue, repo_path, repo_folder) diff --git a/pagure_importer/run.py b/pagure_importer/run.py index a485493..2753298 100644 --- a/pagure_importer/run.py +++ b/pagure_importer/run.py @@ -1,11 +1,35 @@ #!/usr/bin/env python - +import getpass from forms import form_github_issues -from settings import IMPORT_SOURCES, IMPORT_OPTIONS +from settings import IMPORT_SOURCES, IMPORT_OPTIONS, REPO_NAME, REPO_PATH +import pagure_importer +import pagure_importer.lib +import pagure_importer.lib.sources +from pagure_importer.lib.sources.importer_github import GithubImporter +from pagure_importer.lib import generate_json_for_github_contributors, \ + generate_json_for_github_issue_commentors, \ + assemble_github_contributors_commentors def github_handler(item): if item.lower() == 'issues': - form_github_issues() + github_username, github_password, github_project_name = form_github_issues() + gen_json = raw_input('Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') + if gen_json == 'n': + github_importer = GithubImporter( + github_username=github_username, + github_password=github_password, + github_project_name=github_project_name) + github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) + else: + generate_json_for_github_contributors( + github_username, + github_password, + github_project_name) + generate_json_for_github_issue_commentors( + github_username, + github_password, + github_project_name) + assemble_github_contributors_commentors() return def main(): @@ -21,7 +45,6 @@ def main(): if source.lower() == 'github': github_handler(item) - return if __name__ == '__main__': diff --git a/pagure_importer/settings.py b/pagure_importer/settings.py index 13f8332..9761aed 100644 --- a/pagure_importer/settings.py +++ b/pagure_importer/settings.py @@ -1,2 +1,7 @@ +import os + IMPORT_SOURCES = ['github'] IMPORT_OPTIONS = {'github': ['issues']} + +REPO_NAME = os.environ.get('REPO_NAME', None) #this has to be a bare repo +REPO_PATH = os.environ.get('REPO_PATH', None) #the parent of the git directory diff --git a/pagure_importer/sources/__init__.py b/pagure_importer/sources/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/pagure_importer/sources/__init__.py +++ /dev/null diff --git a/pagure_importer/sources/exceptions.py b/pagure_importer/sources/exceptions.py deleted file mode 100644 index 8393fb8..0000000 --- a/pagure_importer/sources/exceptions.py +++ /dev/null @@ -1,9 +0,0 @@ -class GithubBadCredentials(Exception): - ''' Raised when username/password for github is wrong ''' - def __init__(self, msg): - self.msg = msg - -class GithubRepoNotFound(Exception): - ''' Raised when the repo is not found for the user ''' - def __init__(self, msg): - self.msg = msg diff --git a/pagure_importer/sources/importer_github.py b/pagure_importer/sources/importer_github.py deleted file mode 100644 index cc85f7b..0000000 --- a/pagure_importer/sources/importer_github.py +++ /dev/null @@ -1,94 +0,0 @@ -import libpagure -from libpagure.libpagure import Pagure -from github import Github - -from exceptions import GithubBadCredentials, GithubRepoNotFound - - -class GithubImporter(): - ''' Imports from Github using PyGithub and libpagure ''' - - - def __init__( - self, - github_username, - github_password, - github_project_name, - pagure_api_key, - pagure_project_name, - pagure_username=None, - instance_url='https://pagure.io'): - - self.github_username = github_username - self.github_password = github_password - self.github_project_name = github_project_name - self.pagure_project_name = pagure_project_name - self.github = Github(github_username, github_password) - self.pagure = Pagure(pagure_api_key, pagure_project_name, - pagure_username, instance_url) - - - def _get_available_issue_id(self): - ''' Private method which checks the id - which would be available for the new issue - ''' - issues = self.pagure.list_issues() - pull_requests = self.pagure.list_requests() - max_issues = None - max_pull_requests = None - try: - max_issues = max([int(issue['id']) for issue in issues]) - except ValueError: - max_issues = 0 - - try: - max_pull_requests = max([int(pr['id']) for pr in pull_requests]) - except ValueError: - max_pull_requests = 0 - - return max(max_issues, max_pull_requests) + 1 - - - def _get_repo(self, github_user): - ''' Private method to get the repo object - using the given github project name - ''' - repos = github_user.get_repos() - for repo in repos: - if repo.name == self.github_project_name: - return repo - raise GithubRepoNotFound( - 'No user repository with given github project name found') - - - def import_issues(self, status='all'): - ''' Imports the issues on github for - the given project - ''' - github_user = None - try: - github_user = self.github.get_user(self.github_username) - except: - raise GithubBadCredentials( - 'Given github credentials are not correct') - - repo = self._get_repo(github_user) - for github_issue in repo.get_issues(state=status): - pagure_issue_title = github_issue.title - if github_issue.body: - pagure_issue_content = github_issue.body - else: - pagure_issue_content = '#No Description Provided' - - issue_id = self._get_available_issue_id() - self.pagure.create_issue( - pagure_issue_title, pagure_issue_content) - - #comments on the issue - for comment in github_issue.get_comments(): - self.pagure.comment_issue(issue_id, str(comment.body)) - - #change status of the issue if closed - if github_issue.state.lower() == 'closed': - self.pagure.change_issue_status(issue_id, 'Fixed') - From 80300ad87bb0de42075b2a8ab33d9154cb099805 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Apr 10 2016 10:54:46 +0000 Subject: [PATCH 5/24] Readme for the new approach --- diff --git a/README.md b/README.md index b0c883d..46a707d 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,53 @@ # pagure-importer CLI tool for importing issues etc. from different sources like github to pagure -## How to run +## Installation 1. Install it using ```pip``` . ```pip install pagure_importer``` + +## How to run +0. Clone the issue tracker for issues from pagure. Use: ```git clone --bare``` +and set the env variables: 'REPO_NAME' and 'REPO_PATH' +ex: REPO_NAME='abc.git'; REPO_PATH='/home/vivek/' +1. Activate the pagure tickets hook from project settings. 2. Execute ```pgimport``` -3. Just answer what is asked, one by one. +3. Just answer what is asked. Check below instructions for particular source +4. The script will make commits in your cloned bare repo: push the changes back to pagure. ### Present options for sources: github ### Present options for items: issues ### Tools used: -1. [libpagure](https://pagure.io/libpagure) - a python library for [pagure](https://pagure.io) api. -2. [PyGithub](https://github.com/PyGithub/PyGithub) - a python library for [github](https://github.com/) api. +1. [PyGithub](https://github.com/PyGithub/PyGithub) - a python library for [github](https://github.com/) api. + + +## How it works: Github Issues +0. For github issues, there is a bit of pre-processing so, the process is +not very user friendly. The reason behind the pre-processing is that: github +doesn't give away the email ids of issue commentors unless the commentor +is you (if you are logged in) or if the commentor is the issue reporter +himself. So, to overcome this problem, we will be taking email ids from their +commits, if they have contributed to the project but if they haven't, : start +panicking and read below. + +1. We will have to run the script two times. The first time, it will +generate a json file containing all the issue commentors with their details, +if the emails are found, no edit for that particular commentor is required. +Otherwise, you will have to manually fill the emails. Fullnames not required. + +2. After running the program and answering the 'source' and 'items', you +will be asked a question on whether you want to generate a json file for +contributors and issue commentors. If you are running the script for github +for the first time, the answer is 'y'. + +3. The above step will create 3 different json files: ```contributors.json``` +```issue_commentors.json``` and ```assembled_commentors.json```. The last file +is where all the edit has to go. All the missing entries in the assembled +commentors file has to be filled for the running of the script. + +4. Run the script again, filling the same details but answer 'n' when asked for +whether you want to create the json files. In this step, your local issues git +repo gets updated with all the issues from github issue tracker. + +5. Now push the local git repo changes to the remote repo on pagure. It will +update the db and if the user is not found, it will create them from the +details given. From a705a035b4352c7030864fe1bfb593ed998aa08d Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Apr 10 2016 16:19:44 +0000 Subject: [PATCH 6/24] Removed libpagure form requirements --- diff --git a/requirements.txt b/requirements.txt index eeb84d8..52fc450 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -libpagure PyGithub requests From aa46f6b9d0622f2d3bed1f0c7a783555c11de639 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Apr 12 2016 22:17:24 +0000 Subject: [PATCH 7/24] Change the output file to csv type --- diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py index c1724e7..702786c 100644 --- a/pagure_importer/lib/__init__.py +++ b/pagure_importer/lib/__init__.py @@ -1,6 +1,7 @@ import git import models +import csv import os import getpass import requests @@ -13,8 +14,8 @@ from pagure_importer.lib.exceptions import FileNotFound, EmailNotFound def generate_json_for_github_contributors(github_username, github_password, \ github_project_name): - ''' Creates a file containing a list of dicts containing the username and emails - of the contributors in the given github project + ''' Creates a file containing a list of dicts containing the username and + emails of the contributors in the given github project ''' github_obj = Github(github_username, github_password) @@ -26,7 +27,8 @@ def generate_json_for_github_contributors(github_username, github_password, \ while True: page += 1 payload = {'page': page } - data_ = json.loads(requests.get(commits_url, params=payload, auth=HTTPBasicAuth(github_username, github_password)).text) + data_ = json.loads(requests.get(commits_url, params=payload, + auth=HTTPBasicAuth(github_username, github_password)).text) if not data_: break @@ -54,7 +56,7 @@ def generate_json_for_github_contributors(github_username, github_password, \ break if not present: - print 'contributor added: ', len(contributors) + 1 + print 'contributor added: ', contributor_name contributors.append(json_data) with open('contributors.json', 'w') as f: @@ -78,7 +80,8 @@ def generate_json_for_github_issue_commentors(github_username, github_password, while True: page += 1 payload = {'page': page } - data_ = json.loads(requests.get(issue_comment_url, params=payload, auth=HTTPBasicAuth(github_username, github_password)).text) + data_ = json.loads(requests.get(issue_comment_url, params=payload, + auth=HTTPBasicAuth(github_username, github_password)).text) if not data_: break @@ -97,7 +100,7 @@ def generate_json_for_github_issue_commentors(github_username, github_password, break if not present: - print 'commentor added: ', len(issue_commentors) + 1 + print 'commentor added: ', commentor issue_commentors.append(commentor) with open('issue_commentors.json', 'w') as f: @@ -128,8 +131,13 @@ def assemble_github_contributors_commentors(): d = {'name': i, 'fullname': None, 'emails': []} names.append(d) - with open('assembled_commentors.json', 'w') as ac: - json.dump(names, ac) + with open('assembled_commentors.csv', 'w') as ac: + field_names = ['name', 'fullname', 'emails'] + writer = csv.DictWriter(ac, fieldnames=field_names) + + writer.writeheader() + for name in names: + writer.writerow(name) def github_get_commentor_email(name): @@ -137,12 +145,19 @@ def github_get_commentor_email(name): assembled_commentors.json file ''' - if not os.path.exists('assembled_commentors.json'): + if not os.path.exists('assembled_commentors.csv'): raise FileNotFound('The assembled_commentors.json file must be present \ Rerun the program and choose to generate the json files') - with open('assembled_commentors.json') as ac: - data = json.load(ac) + data = [] + with open('assembled_commentors.csv') as ac: + reader = csv.DictReader(ac) + for row in reader: + data.append(dict( \ + (('name', row['name']), \ + ('fullname', row['fullname']), \ + ('emails', row['emails'])))) + for i in data: if i.get('name', None) == name: From a7ec751d7eec0ca88d6b943e81f2d23ca37cb7b1 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Apr 12 2016 22:17:44 +0000 Subject: [PATCH 8/24] Corrected import --- diff --git a/pagure_importer/forms.py b/pagure_importer/forms.py index ddc24f0..a111114 100644 --- a/pagure_importer/forms.py +++ b/pagure_importer/forms.py @@ -1,6 +1,6 @@ import getpass -from lib.sources.importer_github_new import GithubImporter +from lib.sources.importer_github import GithubImporter from settings import REPO_PATH, REPO_NAME def form_github_issues(): From ebd70ed42fd9b68b0da45468dd4ac9cdee7d07ec Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Apr 14 2016 07:50:21 +0000 Subject: [PATCH 9/24] Email as a string in csv --- diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py index 702786c..91e0dd4 100644 --- a/pagure_importer/lib/__init__.py +++ b/pagure_importer/lib/__init__.py @@ -124,11 +124,12 @@ def assemble_github_contributors_commentors(): found = False for j in contributors: if j.get('name', None) == i: + j['emails'] = j.get('emails')[0] names.append(j) found = True if not found: - d = {'name': i, 'fullname': None, 'emails': []} + d = {'name': i, 'fullname': None, 'emails': None} names.append(d) with open('assembled_commentors.csv', 'w') as ac: @@ -142,7 +143,7 @@ def assemble_github_contributors_commentors(): def github_get_commentor_email(name): ''' Will return the issue commentor email as given in the - assembled_commentors.json file + assembled_commentors.csv file ''' if not os.path.exists('assembled_commentors.csv'): @@ -162,7 +163,8 @@ def github_get_commentor_email(name): for i in data: if i.get('name', None) == name: if i['emails']: - return i['emails'] + return str(i['emails']) else: raise EmailNotFound('You need to fill out all the emails of the \ issue commentors') + diff --git a/pagure_importer/lib/sources/importer_github.py b/pagure_importer/lib/sources/importer_github.py index 2e635b6..2859e99 100644 --- a/pagure_importer/lib/sources/importer_github.py +++ b/pagure_importer/lib/sources/importer_github.py @@ -112,7 +112,8 @@ class GithubImporter(): pagure_issue_comment_user = models.User( name=comment_user.login, fullname=comment_user.name, - emails=[comment_user.email] if comment_user.email else github_get_commentor_email(comment_user.login)) + emails=[comment_user.email] if comment_user.email \ + else [github_get_commentor_email(comment_user.login)]) #Object to represent comment on an issue pagure_issue_comment = models.IssueComment( From 52ab30a6b7659a93e8e80e52e417735d173f1318 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Apr 14 2016 08:04:53 +0000 Subject: [PATCH 10/24] reflect change in output file name in Readme --- diff --git a/README.md b/README.md index 46a707d..b2075e2 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ CLI tool for importing issues etc. from different sources like github to pagure ## How to run 0. Clone the issue tracker for issues from pagure. Use: ```git clone --bare``` -and set the env variables: 'REPO_NAME' and 'REPO_PATH' -ex: REPO_NAME='abc.git'; REPO_PATH='/home/vivek/' -1. Activate the pagure tickets hook from project settings. -2. Execute ```pgimport``` -3. Just answer what is asked. Check below instructions for particular source -4. The script will make commits in your cloned bare repo: push the changes back to pagure. +1. set the env variables: ```REPO_NAME``` and ```REPO_PATH``` +ex: REPO_NAME=abc.git; REPO_PATH=/home/vivek/ +2. Activate the pagure tickets hook from project settings. +3. Execute ```pgimport``` +4. Just answer what is asked. Check below instructions for particular source +5. The script will make commits in your cloned bare repo: push the changes back to pagure. ### Present options for sources: github ### Present options for items: issues @@ -39,8 +39,8 @@ will be asked a question on whether you want to generate a json file for contributors and issue commentors. If you are running the script for github for the first time, the answer is 'y'. -3. The above step will create 3 different json files: ```contributors.json``` -```issue_commentors.json``` and ```assembled_commentors.json```. The last file +3. The above step will create 3 different files: ```contributors.json``` +```issue_commentors.json``` and ```assembled_commentors.csv```. The last file is where all the edit has to go. All the missing entries in the assembled commentors file has to be filled for the running of the script. From 4a6bb8177ace90afbe44980c5b08725e6011a543 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Apr 14 2016 08:22:23 +0000 Subject: [PATCH 11/24] Doc string corrected for a func --- diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py index 91e0dd4..9a64e3b 100644 --- a/pagure_importer/lib/__init__.py +++ b/pagure_importer/lib/__init__.py @@ -110,8 +110,8 @@ def generate_json_for_github_issue_commentors(github_username, github_password, def assemble_github_contributors_commentors(): ''' It uses the files: issue_commentors.json and contributors.json - Assembles and creates a file: assembled_commentors.json - To use: just fill the None and [] in the final file ''' + Assembles and creates a file: assembled_commentors.csv + To use: just fill the empty blocks under emails column''' with open('issue_commentors.json', 'r') as ic: issue_names = json.load(ic) From a6c253646b80a70cd07799585dba0d19d269cad0 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Apr 15 2016 19:26:00 +0000 Subject: [PATCH 12/24] trac importer skeleton --- diff --git a/pagure_importer/lib/sources/importer_trac.py b/pagure_importer/lib/sources/importer_trac.py new file mode 100644 index 0000000..3ff00a7 --- /dev/null +++ b/pagure_importer/lib/sources/importer_trac.py @@ -0,0 +1,106 @@ +from xmlrpclib import ServerProxy +import pagure_importer +import pagure_importer.lib +from pagure_importer.lib import models +from datetime import datetime + +class TracImporter(): + '''Pagure importer for trac instance''' + + def __init__(self, trac_project_url): + self.trac = ServerProxy(trac_project_url + '/rpc') + + def import_issues(self, repo_path, repo_folder, trac_query='report=9&order=id'): + '''Import issues from trac instance using xmlrpc API''' + tickets_list = [] + tickets_id = self.trac.ticket.query(trac_query) + + for ticket_id in tickets_id: + + trac_ticket = self.trac.ticket.get(ticket_id)[3] + pagure_issue_title = trac_ticket['summary'] + pagure_issue_content = trac_ticket['description'] + + if pagure_issue_content == '': + pagure_issue_content = '#No Description Provided' + + if trac_ticket['status'] != 'closed': + pagure_issue_status = 'Open' + else: + pagure_issue_status = 'Fixed' + + pagure_issue_created_at = datetime.strptime(self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") + + pagure_issue_assignee = trac_ticket['owner'] + + # if github_issue.labels: + # pagure_issue_tags = [i.name for i in github_issue.labels] + # else: + pagure_issue_tags = [] + + pagure_issue_depends = [] + pagure_issue_blocks = [] + pagure_issue_is_private = False + + pagure_issue_user = models.User( + name=trac_ticket['reporter'], + fullname=None, + emails=[None]) + + pagure_issue = models.Issue( + id=None, + title=pagure_issue_title, + content=pagure_issue_content, + status=pagure_issue_status, + date_created=pagure_issue_created_at, + user=pagure_issue_user.to_json(), + private=pagure_issue_is_private, + tags=pagure_issue_tags, + depends=pagure_issue_depends, + blocks=pagure_issue_blocks, + assignee=pagure_issue_assignee) + + pagure_issue_comments = self.trac.ticket.changeLog(ticket_id) + comments = [] + for comment in pagure_issue_comments: + if comment[2] == 'comment' and comment[4] != '': + comment_user = comment[1] + pagure_issue_comment_user_email = None + pagure_issue_comment_body = comment[4] + pagure_issue_comment_created_at = datetime.strptime(comment[0].value, "%Y%m%dT%H:%M:%S") + pagure_issue_comment_updated_at = None + + # No idea what to do with this right now + # editor: not supported by github api + pagure_issue_comment_parent = None + pagure_issue_comment_editor = None + + # comment updated at + pagure_issue_comment_edited_on = None + + # The User who commented + pagure_issue_comment_user = models.User( + name=comment[1], + fullname=None, + emails=None) + + # Object to represent comment on an issue + pagure_issue_comment = models.IssueComment( + id=None, + comment=pagure_issue_comment_body, + parent=pagure_issue_comment_parent, + date_created=pagure_issue_comment_created_at, + user=pagure_issue_comment_user.to_json(), + edited_on=pagure_issue_comment_edited_on, + editor=pagure_issue_comment_editor) + + comments.append(pagure_issue_comment.to_json()) + + # add all the comments to the issue object + pagure_issue.comments = comments + + # update the local git repo + print 'Update repo with issue :' + str(ticket_id) + pagure_importer.lib.git.update_git(pagure_issue, + repo_path, + repo_folder) diff --git a/pagure_importer/run.py b/pagure_importer/run.py index 2753298..d327228 100644 --- a/pagure_importer/run.py +++ b/pagure_importer/run.py @@ -6,9 +6,11 @@ import pagure_importer import pagure_importer.lib import pagure_importer.lib.sources from pagure_importer.lib.sources.importer_github import GithubImporter +from pagure_importer.lib.sources.importer_trac import TracImporter from pagure_importer.lib import generate_json_for_github_contributors, \ - generate_json_for_github_issue_commentors, \ - assemble_github_contributors_commentors + generate_json_for_github_issue_commentors, \ + assemble_github_contributors_commentors + def github_handler(item): if item.lower() == 'issues': @@ -16,22 +18,30 @@ def github_handler(item): gen_json = raw_input('Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') if gen_json == 'n': github_importer = GithubImporter( - github_username=github_username, - github_password=github_password, - github_project_name=github_project_name) + github_username=github_username, + github_password=github_password, + github_project_name=github_project_name) github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) else: generate_json_for_github_contributors( - github_username, - github_password, - github_project_name) + github_username, + github_password, + github_project_name) generate_json_for_github_issue_commentors( - github_username, - github_password, - github_project_name) + github_username, + github_password, + github_project_name) assemble_github_contributors_commentors() return + +def trac_handler(item, fedora=False): + if item.lower() == 'issues': + trac_url = raw_input('Enter the trac project url: ') + trac_importer = TracImporter(trac_url) + trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) + + def main(): source = raw_input('Enter source from where you want to import: ') if source.lower() not in IMPORT_SOURCES: @@ -45,6 +55,8 @@ def main(): if source.lower() == 'github': github_handler(item) + elif source.lower() == 'fedorahosted': + trac_handler(item, fedora=True) return if __name__ == '__main__': diff --git a/pagure_importer/settings.py b/pagure_importer/settings.py index 9761aed..077ea40 100644 --- a/pagure_importer/settings.py +++ b/pagure_importer/settings.py @@ -1,7 +1,7 @@ import os -IMPORT_SOURCES = ['github'] -IMPORT_OPTIONS = {'github': ['issues']} +IMPORT_SOURCES = ['github', 'fedorahosted'] +IMPORT_OPTIONS = {'github': ['issues'], 'fedorahosted': ['issues']} REPO_NAME = os.environ.get('REPO_NAME', None) #this has to be a bare repo REPO_PATH = os.environ.get('REPO_PATH', None) #the parent of the git directory From b51b561f8ffadae50c804df7a0b43620a3617867 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Apr 16 2016 21:07:24 +0000 Subject: [PATCH 13/24] Refactor trac importer for more flexibility --- diff --git a/pagure_importer/lib/sources/importer_trac.py b/pagure_importer/lib/sources/importer_trac.py index 3ff00a7..2e6fce2 100644 --- a/pagure_importer/lib/sources/importer_trac.py +++ b/pagure_importer/lib/sources/importer_trac.py @@ -2,104 +2,132 @@ from xmlrpclib import ServerProxy import pagure_importer import pagure_importer.lib from pagure_importer.lib import models +from fedora.client.fas2 import AccountSystem from datetime import datetime + class TracImporter(): '''Pagure importer for trac instance''' def __init__(self, trac_project_url): self.trac = ServerProxy(trac_project_url + '/rpc') - - def import_issues(self, repo_path, repo_folder, trac_query='report=9&order=id'): + fas_url = 'https://admin.fedoraproject.org/accounts' + fas_username = 'user' + fas_password = 'pass' + self.fasclient = AccountSystem(fas_url, username=fas_username, + password=fas_password) + + def _find_fas_user(self, user): + person = self.fasclient.person_by_username(user) + human_name = person['human_name'] + email = person['email'] + pagure_user = models.User( + name=user, + fullname=human_name, + emails=[email]) + return pagure_user + + def _get_ticket_tags(self, trac_ticket): + return [] + + def _get_ticket_status(self, trac_ticket): + ''' Converts Trac ticket status + to Pagure issue status''' + + if trac_ticket['status'] != 'closed': + ticket_status = 'Open' + else: + ticket_status = 'Fixed' + return ticket_status + + def _populate_comments(self, trac_comments): + comments = [] + for comment in trac_comments: + if comment[2] == 'comment' and comment[4] != '': + comment_user = comment[1] + pagure_issue_comment_user_email = None + pagure_issue_comment_body = comment[4] + pagure_issue_comment_created_at = datetime.strptime( + comment[0].value, "%Y%m%dT%H:%M:%S") + pagure_issue_comment_updated_at = None + + # No idea what to do with this right now + # editor: not supported by github api + pagure_issue_comment_parent = None + pagure_issue_comment_editor = None + + # comment updated at + pagure_issue_comment_edited_on = None + + # The User who commented + pagure_issue_comment_user = self._find_fas_user(comment[1]) + + # Object to represent comment on an issue + pagure_issue_comment = models.IssueComment( + id=None, + comment=pagure_issue_comment_body, + parent=pagure_issue_comment_parent, + date_created=pagure_issue_comment_created_at, + user=pagure_issue_comment_user.to_json(), + edited_on=pagure_issue_comment_edited_on, + editor=pagure_issue_comment_editor) + + comments.append(pagure_issue_comment.to_json()) + return comments + + def _populate_issue(self, ticket_id): + trac_ticket = self.trac.ticket.get(ticket_id)[3] + pagure_issue_title = trac_ticket['summary'] + pagure_issue_content = trac_ticket['description'] + + if pagure_issue_content == '': + pagure_issue_content = '#No Description Provided' + + pagure_issue_status = self._get_ticket_status(trac_ticket) + + pagure_issue_created_at = datetime.strptime( + self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") + + pagure_issue_assignee = trac_ticket['owner'] + + pagure_issue_tags = self._get_ticket_tags(trac_ticket) + + pagure_issue_depends = [] + pagure_issue_blocks = [] + pagure_issue_is_private = False + + pagure_issue_user = self._find_fas_user(trac_ticket['reporter']) + + pagure_issue = models.Issue( + id=None, + title=pagure_issue_title, + content=pagure_issue_content, + status=pagure_issue_status, + date_created=pagure_issue_created_at, + user=pagure_issue_user.to_json(), + private=pagure_issue_is_private, + tags=pagure_issue_tags, + depends=pagure_issue_depends, + blocks=pagure_issue_blocks, + assignee=pagure_issue_assignee) + return pagure_issue + + def import_issues(self, repo_path, repo_folder, + trac_query='max=0&order=id'): '''Import issues from trac instance using xmlrpc API''' - tickets_list = [] tickets_id = self.trac.ticket.query(trac_query) for ticket_id in tickets_id: - trac_ticket = self.trac.ticket.get(ticket_id)[3] - pagure_issue_title = trac_ticket['summary'] - pagure_issue_content = trac_ticket['description'] - - if pagure_issue_content == '': - pagure_issue_content = '#No Description Provided' - - if trac_ticket['status'] != 'closed': - pagure_issue_status = 'Open' - else: - pagure_issue_status = 'Fixed' - - pagure_issue_created_at = datetime.strptime(self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") - - pagure_issue_assignee = trac_ticket['owner'] - - # if github_issue.labels: - # pagure_issue_tags = [i.name for i in github_issue.labels] - # else: - pagure_issue_tags = [] - - pagure_issue_depends = [] - pagure_issue_blocks = [] - pagure_issue_is_private = False - - pagure_issue_user = models.User( - name=trac_ticket['reporter'], - fullname=None, - emails=[None]) - - pagure_issue = models.Issue( - id=None, - title=pagure_issue_title, - content=pagure_issue_content, - status=pagure_issue_status, - date_created=pagure_issue_created_at, - user=pagure_issue_user.to_json(), - private=pagure_issue_is_private, - tags=pagure_issue_tags, - depends=pagure_issue_depends, - blocks=pagure_issue_blocks, - assignee=pagure_issue_assignee) + pagure_issue = self._populate_issue(ticket_id) pagure_issue_comments = self.trac.ticket.changeLog(ticket_id) - comments = [] - for comment in pagure_issue_comments: - if comment[2] == 'comment' and comment[4] != '': - comment_user = comment[1] - pagure_issue_comment_user_email = None - pagure_issue_comment_body = comment[4] - pagure_issue_comment_created_at = datetime.strptime(comment[0].value, "%Y%m%dT%H:%M:%S") - pagure_issue_comment_updated_at = None - - # No idea what to do with this right now - # editor: not supported by github api - pagure_issue_comment_parent = None - pagure_issue_comment_editor = None - - # comment updated at - pagure_issue_comment_edited_on = None - - # The User who commented - pagure_issue_comment_user = models.User( - name=comment[1], - fullname=None, - emails=None) - - # Object to represent comment on an issue - pagure_issue_comment = models.IssueComment( - id=None, - comment=pagure_issue_comment_body, - parent=pagure_issue_comment_parent, - date_created=pagure_issue_comment_created_at, - user=pagure_issue_comment_user.to_json(), - edited_on=pagure_issue_comment_edited_on, - editor=pagure_issue_comment_editor) - - comments.append(pagure_issue_comment.to_json()) - - # add all the comments to the issue object - pagure_issue.comments = comments - - # update the local git repo + comments = self._populate_comments(pagure_issue_comments) + + # add all the comments to the issue object + pagure_issue.comments = comments + + # update the local git repo print 'Update repo with issue :' + str(ticket_id) pagure_importer.lib.git.update_git(pagure_issue, repo_path, From 404bfcc5e365dbc1dd26fe3b0ae71f53387f81df Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Apr 22 2016 08:39:24 +0000 Subject: [PATCH 14/24] gitignore changes --- diff --git a/.gitignore b/.gitignore index ac43ebc..358cdf2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,11 @@ +dist/ +build/ *.pyc *.py~ +*.pyo +*.swp +*.egg-info +*.conf +.coverage +*.json +*.csv From 0cfa26bc59668717d954f5673438337597f671ee Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Apr 22 2016 08:45:09 +0000 Subject: [PATCH 15/24] FAS and TRAC support added to importer --- diff --git a/pagure_importer/lib/fas.py b/pagure_importer/lib/fas.py new file mode 100644 index 0000000..b63f665 --- /dev/null +++ b/pagure_importer/lib/fas.py @@ -0,0 +1,23 @@ +from fedora.client.fas2 import AccountSystem +from pagure_importer.lib.models import User + + +class FASclient (): + def __init__(self, fas_username, fas_password, fas_url): + self.fasclient = AccountSystem(fas_url, username=fas_username, + password=fas_password) + + anonymous = User(name='', fullname='', emails=[]) + self.fasuser = {'': anonymous} + + def find_fas_user(self, user): + + if user not in self.fasuser.keys(): + person = self.fasclient.person_by_username(user) + if not person: + return self.fasuser[''] + + self.fasuser[user] = User(name=user, + fullname=person['human_name'], + emails=[person['email']]) + return self.fasuser[user] diff --git a/pagure_importer/lib/sources/importer_trac.py b/pagure_importer/lib/sources/importer_trac.py index 2e6fce2..f69a3ce 100644 --- a/pagure_importer/lib/sources/importer_trac.py +++ b/pagure_importer/lib/sources/importer_trac.py @@ -1,21 +1,17 @@ from xmlrpclib import ServerProxy import pagure_importer import pagure_importer.lib -from pagure_importer.lib import models -from fedora.client.fas2 import AccountSystem -from datetime import datetime +from pagure_importer.lib.fas import FASclient +from pagure_importer.lib import trac class TracImporter(): '''Pagure importer for trac instance''' def __init__(self, trac_project_url): - self.trac = ServerProxy(trac_project_url + '/rpc') - fas_url = 'https://admin.fedoraproject.org/accounts' - fas_username = 'user' - fas_password = 'pass' - self.fasclient = AccountSystem(fas_url, username=fas_username, - password=fas_password) + self.tracclient = ServerProxy(trac_project_url + '/rpc') + self.fasclient = FASclient('user', 'password', + 'https://admin.fedoraproject.org/accounts') def _find_fas_user(self, user): person = self.fasclient.person_by_username(user) @@ -115,14 +111,16 @@ class TracImporter(): def import_issues(self, repo_path, repo_folder, trac_query='max=0&order=id'): '''Import issues from trac instance using xmlrpc API''' - tickets_id = self.trac.ticket.query(trac_query) + tickets_id = self.tracclient.ticket.query(trac_query) for ticket_id in tickets_id: - pagure_issue = self._populate_issue(ticket_id) + pagure_issue = trac.populate_issue(self.tracclient, + self.fasclient, ticket_id) - pagure_issue_comments = self.trac.ticket.changeLog(ticket_id) - comments = self._populate_comments(pagure_issue_comments) + pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) + comments = trac.populate_comments(self.fasclient, + pagure_issue_comments) # add all the comments to the issue object pagure_issue.comments = comments diff --git a/pagure_importer/lib/trac.py b/pagure_importer/lib/trac.py new file mode 100644 index 0000000..fb3d062 --- /dev/null +++ b/pagure_importer/lib/trac.py @@ -0,0 +1,90 @@ +from pagure_importer.lib.models import IssueComment, Issue +from datetime import datetime + + +def get_ticket_tags(trac_ticket): + return [] + + +def get_ticket_status(trac_ticket): + ''' Converts Trac ticket status + to Pagure issue status''' + + if trac_ticket['status'] != 'closed': + ticket_status = 'Open' + else: + ticket_status = 'Fixed' + return ticket_status + + +def populate_comments(fasclient, trac_comments): + comments = [] + for comment in trac_comments: + if comment[2] == 'comment' and comment[4] != '': + comment_user = comment[1] + pagure_issue_comment_user_email = None + pagure_issue_comment_body = comment[4] + pagure_issue_comment_created_at = datetime.strptime( + comment[0].value, "%Y%m%dT%H:%M:%S") + pagure_issue_comment_updated_at = None + + # No idea what to do with this right now + # editor: not supported by github api + pagure_issue_comment_parent = None + pagure_issue_comment_editor = None + + # comment updated at + pagure_issue_comment_edited_on = None + + # The User who commented + pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) + + # Object to represent comment on an issue + pagure_issue_comment = IssueComment( + id=None, + comment=pagure_issue_comment_body, + parent=pagure_issue_comment_parent, + date_created=pagure_issue_comment_created_at, + user=pagure_issue_comment_user.to_json(), + edited_on=pagure_issue_comment_edited_on, + editor=pagure_issue_comment_editor) + + comments.append(pagure_issue_comment.to_json()) + return comments + + +def populate_issue(trac, fasclient, ticket_id): + trac_ticket = trac.ticket.get(ticket_id)[3] + pagure_issue_title = trac_ticket['summary'] + pagure_issue_content = trac_ticket['description'] + + if pagure_issue_content == '': + pagure_issue_content = '#No Description Provided' + + pagure_issue_status = get_ticket_status(trac_ticket) + + pagure_issue_created_at = datetime.strptime( + trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") + + pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) + + pagure_issue_tags = get_ticket_tags(trac_ticket) + + pagure_issue_depends = [] + pagure_issue_blocks = [] + pagure_issue_is_private = False + + pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) + pagure_issue = Issue( + id=ticket_id, + title=pagure_issue_title, + content=pagure_issue_content, + status=pagure_issue_status, + date_created=pagure_issue_created_at, + user=pagure_issue_user.to_json(), + private=pagure_issue_is_private, + tags=pagure_issue_tags, + depends=pagure_issue_depends, + blocks=pagure_issue_blocks, + assignee=pagure_issue_assignee.to_json()) + return pagure_issue From 84add1511833031f88b5c16f9c2b6796f37be456 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 04 2016 19:13:36 +0000 Subject: [PATCH 16/24] Reworked app structure to use click cli framework. App structure base on https://pagure.io/pag --- diff --git a/diff b/diff new file mode 100644 index 0000000..2a92e6e --- /dev/null +++ b/diff @@ -0,0 +1,1922 @@ +diff --git a/pagure_importer/__init__.py b/pagure_importer/__init__.py +index fbab797..8b13789 100644 +--- a/pagure_importer/__init__.py ++++ b/pagure_importer/__init__.py +@@ -1,2 +1 @@ +-import lib +-import settings ++ +diff --git a/pagure_importer/app.py b/pagure_importer/app.py +new file mode 100644 +index 0000000..6dc3277 +--- /dev/null ++++ b/pagure_importer/app.py +@@ -0,0 +1,23 @@ ++#!/usr/bin/env python ++ ++import click ++import os ++ ++REPO_NAME = os.environ.get('REPO_NAME', None) # this has to be a bare repo ++REPO_PATH = os.environ.get('REPO_PATH', None) # the parent of the git directory ++ ++ ++@click.group() ++def app(): ++ pass ++ ++__all__ = [ ++ 'app', ++] ++ ++# from .commands import github ++from .commands import fedorahosted ++from .commands import github ++ ++if __name__ == '__main__': ++ app() +diff --git a/pagure_importer/commands/__init__.py b/pagure_importer/commands/__init__.py +new file mode 100644 +index 0000000..e69de29 +diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py +new file mode 100644 +index 0000000..650d133 +--- /dev/null ++++ b/pagure_importer/commands/fedorahosted.py +@@ -0,0 +1,15 @@ ++import click ++import getpass ++from pagure_importer.app import app, REPO_NAME, REPO_PATH ++from pagure_importer.utils import importer_trac ++from pagure_importer.utils.fas import FASclient ++ ++@app.command() ++@click.argument('project_url') ++def fedorahosted(project_url): ++ fas_username = raw_input('Enter you Github Username: ') ++ fas_password = getpass.getpass('Enter your github password: ') ++ fasclient = FASclient(fas_username, fas_password, ++ 'https://admin.fedoraproject.org/accounts') ++ trac_importer = importer_trac.TracImporter(project_url, fasclient) ++ trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) +diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py +new file mode 100644 +index 0000000..26ae4fa +--- /dev/null ++++ b/pagure_importer/commands/github.py +@@ -0,0 +1,40 @@ ++import click ++import getpass ++ ++from pagure_importer.app import app, REPO_NAME, REPO_PATH ++from pagure_importer.utils.importer_github import GithubImporter ++from pagure_importer.utils import ( ++ generate_json_for_github_contributors, ++ generate_json_for_github_issue_commentors, ++ assemble_github_contributors_commentors ++) ++ ++ ++def form_github_issues(): ++ github_username = raw_input('Enter you Github Username: ') ++ github_password = getpass.getpass('Enter your github password: ') ++ github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') ++ return (github_username, github_password, github_project_name) ++ ++@app.command() ++def github(): ++ github_username, github_password, github_project_name = form_github_issues() ++ gen_json = raw_input( ++ 'Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') ++ if gen_json == 'n': ++ github_importer = GithubImporter( ++ github_username=github_username, ++ github_password=github_password, ++ github_project_name=github_project_name) ++ github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) ++ else: ++ generate_json_for_github_contributors( ++ github_username, ++ github_password, ++ github_project_name) ++ generate_json_for_github_issue_commentors( ++ github_username, ++ github_password, ++ github_project_name) ++ assemble_github_contributors_commentors() ++ return +diff --git a/pagure_importer/forms.py b/pagure_importer/forms.py +deleted file mode 100644 +index a111114..0000000 +--- a/pagure_importer/forms.py ++++ /dev/null +@@ -1,10 +0,0 @@ +-import getpass +- +-from lib.sources.importer_github import GithubImporter +-from settings import REPO_PATH, REPO_NAME +- +-def form_github_issues(): +- github_username = raw_input('Enter you Github Username: ') +- github_password = getpass.getpass('Enter your github password: ') +- github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') +- return (github_username, github_password, github_project_name) +diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py +deleted file mode 100644 +index 9a64e3b..0000000 +--- a/pagure_importer/lib/__init__.py ++++ /dev/null +@@ -1,170 +0,0 @@ +-import git +-import models +- +-import csv +-import os +-import getpass +-import requests +-import json +-from github import Github +-from requests.auth import HTTPBasicAuth +-import pagure_importer +-import pagure_importer.lib +-from pagure_importer.lib.exceptions import FileNotFound, EmailNotFound +- +-def generate_json_for_github_contributors(github_username, github_password, \ +- github_project_name): +- ''' Creates a file containing a list of dicts containing the username and +- emails of the contributors in the given github project +- ''' +- +- github_obj = Github(github_username, github_password) +- project = github_obj.get_repo(github_project_name) +- commits_url = project.commits_url.replace('{/sha}', '') +- +- page = 0 +- contributors = [] +- while True: +- page += 1 +- payload = {'page': page } +- data_ = json.loads(requests.get(commits_url, params=payload, +- auth=HTTPBasicAuth(github_username, github_password)).text) +- +- if not data_: +- break +- +- for data in data_: +- try: +- contributor = data['commit']['committer'] +- contributor_email = contributor['email'] +- contributor_fullname = contributor['name'] +- contributor_name = data['committer']['login'] +- except TypeError: +- print 'Maybe one of the contributors is dropped because of lack of details' +- continue +- +- json_data = { +- 'name': contributor_name, +- 'fullname': contributor_fullname, +- 'emails': [contributor_email] +- } +- +- present = False +- for i in contributors: +- if i == json_data: +- present = True +- break +- +- if not present: +- print 'contributor added: ', contributor_name +- contributors.append(json_data) +- +- with open('contributors.json', 'w') as f: +- f.write(json.dumps(contributors)) +- +- return +- +- +-def generate_json_for_github_issue_commentors(github_username, github_password, \ +- github_project_name): +- ''' Will create a json file containing details of all the user +- who have commented on any issue in the given project +- ''' +- +- github_obj = Github(github_username, github_password) +- project = github_obj.get_repo(github_project_name) +- issue_comment_url = project.issue_comment_url.replace('{/number}', '') +- +- page = 0 +- issue_commentors = [] +- while True: +- page += 1 +- payload = {'page': page } +- data_ = json.loads(requests.get(issue_comment_url, params=payload, +- auth=HTTPBasicAuth(github_username, github_password)).text) +- +- if not data_: +- break +- +- for data in data_: +- try: +- commentor = data['user']['login'] +- except TypeError: +- print 'Maybe one of the issue commentors have been dropped because of lack of details' +- continue +- +- present = False +- for i in issue_commentors: +- if i == commentor: +- present = True +- break +- +- if not present: +- print 'commentor added: ', commentor +- issue_commentors.append(commentor) +- +- with open('issue_commentors.json', 'w') as f: +- f.write(json.dumps(issue_commentors)) +- return +- +- +-def assemble_github_contributors_commentors(): +- ''' It uses the files: issue_commentors.json and contributors.json +- Assembles and creates a file: assembled_commentors.csv +- To use: just fill the empty blocks under emails column''' +- +- with open('issue_commentors.json', 'r') as ic: +- issue_names = json.load(ic) +- +- with open('contributors.json', 'r') as c: +- contributors = json.load(c) +- +- names = [] +- for i in issue_names: +- found = False +- for j in contributors: +- if j.get('name', None) == i: +- j['emails'] = j.get('emails')[0] +- names.append(j) +- found = True +- +- if not found: +- d = {'name': i, 'fullname': None, 'emails': None} +- names.append(d) +- +- with open('assembled_commentors.csv', 'w') as ac: +- field_names = ['name', 'fullname', 'emails'] +- writer = csv.DictWriter(ac, fieldnames=field_names) +- +- writer.writeheader() +- for name in names: +- writer.writerow(name) +- +- +-def github_get_commentor_email(name): +- ''' Will return the issue commentor email as given in the +- assembled_commentors.csv file +- ''' +- +- if not os.path.exists('assembled_commentors.csv'): +- raise FileNotFound('The assembled_commentors.json file must be present \ +- Rerun the program and choose to generate the json files') +- +- data = [] +- with open('assembled_commentors.csv') as ac: +- reader = csv.DictReader(ac) +- for row in reader: +- data.append(dict( \ +- (('name', row['name']), \ +- ('fullname', row['fullname']), \ +- ('emails', row['emails'])))) +- +- +- for i in data: +- if i.get('name', None) == name: +- if i['emails']: +- return str(i['emails']) +- else: +- raise EmailNotFound('You need to fill out all the emails of the \ +- issue commentors') +- +diff --git a/pagure_importer/lib/exceptions.py b/pagure_importer/lib/exceptions.py +deleted file mode 100644 +index a5ab5fe..0000000 +--- a/pagure_importer/lib/exceptions.py ++++ /dev/null +@@ -1,22 +0,0 @@ +-class GithubBadCredentials(Exception): +- ''' Raised when username/password for github is wrong ''' +- def __init__(self, msg): +- self.msg = msg +- +- +-class GithubRepoNotFound(Exception): +- ''' Raised when the repo is not found for the user ''' +- def __init__(self, msg): +- self.msg = msg +- +- +-class FileNotFound(Exception): +- ''' Raised when a certain file is not found ''' +- def __init__(self, msg): +- self.msg = msg +- +- +-class EmailNotFound(Exception): +- ''' Raised when email is not found ''' +- def __init__(self, msg): +- self.msg = msg +diff --git a/pagure_importer/lib/fas.py b/pagure_importer/lib/fas.py +deleted file mode 100644 +index b63f665..0000000 +--- a/pagure_importer/lib/fas.py ++++ /dev/null +@@ -1,23 +0,0 @@ +-from fedora.client.fas2 import AccountSystem +-from pagure_importer.lib.models import User +- +- +-class FASclient (): +- def __init__(self, fas_username, fas_password, fas_url): +- self.fasclient = AccountSystem(fas_url, username=fas_username, +- password=fas_password) +- +- anonymous = User(name='', fullname='', emails=[]) +- self.fasuser = {'': anonymous} +- +- def find_fas_user(self, user): +- +- if user not in self.fasuser.keys(): +- person = self.fasclient.person_by_username(user) +- if not person: +- return self.fasuser[''] +- +- self.fasuser[user] = User(name=user, +- fullname=person['human_name'], +- emails=[person['email']]) +- return self.fasuser[user] +diff --git a/pagure_importer/lib/git.py b/pagure_importer/lib/git.py +deleted file mode 100644 +index 33eba48..0000000 +--- a/pagure_importer/lib/git.py ++++ /dev/null +@@ -1,98 +0,0 @@ +-''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py +- by pingou@pingoured.fr +-''' +- +-import shutil +-import os +-import pygit2 +-import tempfile +-import json +- +-from repo import * +- +-def update_git(obj, repo_path, repo_folder): +- """ Update the given issue in its git. +- This method forks the provided repo, add/edit the issue whose file name +- is defined by the uid field of the issue and if there are additions/ +- changes commit them and push them back to the original repo. +- """ +- +- if not repo_folder: +- return +- +- # Get the fork +- repopath = os.path.join(repo_folder, repo_path) +- +- # Clone the repo into a temp folder +- newpath = tempfile.mkdtemp(prefix='pagure-') +- new_repo = pygit2.clone_repository(repopath, newpath) +- +- file_path = os.path.join(newpath, obj.uid) +- +- # Get the current index +- index = new_repo.index +- +- # Are we adding files +- added = False +- if not os.path.exists(file_path): +- added = True +- +- # Write down what changed +- with open(file_path, 'w') as stream: +- stream.write(json.dumps( +- obj.to_json(), sort_keys=True, indent=4, +- separators=(',', ': '))) +- +- # Retrieve the list of files that changed +- diff = new_repo.diff() +- files = [] +- for p in diff: +- if hasattr(p, 'new_file_path'): +- files.append(p.new_file_path) +- elif hasattr(p, 'delta'): +- files.append(p.delta.new_file.path) +- +- # Add the changes to the index +- if added: +- index.add(obj.uid) +- for filename in files: +- index.add(filename) +- +- # If not change, return +- if not files and not added: +- shutil.rmtree(newpath) +- return +- +- # See if there is a parent to this commit +- parent = None +- try: +- parent = new_repo.head.get_object().oid +- except pygit2.GitError: +- pass +- +- parents = [] +- if parent: +- parents.append(parent) +- +- # Author/commiter will always be this one +- author = pygit2.Signature(name='pagure', email='pagure') +- +- # Actually commit +- new_repo.create_commit( +- 'refs/heads/master', +- author, +- author, +- 'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title), +- new_repo.index.write_tree(), +- parents) +- index.write() +- +- # Push to origin +- ori_remote = new_repo.remotes[0] +- master_ref = new_repo.lookup_reference('HEAD').resolve() +- refname = '%s:%s' % (master_ref.name, master_ref.name) +- +- PagureRepo.push(ori_remote, refname) +- +- # Remove the clone +- shutil.rmtree(newpath) +diff --git a/pagure_importer/lib/models.py b/pagure_importer/lib/models.py +deleted file mode 100644 +index 3aa3d83..0000000 +--- a/pagure_importer/lib/models.py ++++ /dev/null +@@ -1,105 +0,0 @@ +-# -*- coding: utf-8 -*- +- +-import datetime +-import json +-import uuid +- +-class Issue(): +- ''' Represents an Issue ''' +- +- def __init__( +- self, id, title, content, +- status, date_created, user, private, tags, +- depends, blocks, assignee, comments=None): +- +- self.id = id +- self.title = title +- self.content = content +- self.status = status +- self.date_created = date_created +- self.user = user +- self.private = private +- self.tags = tags +- self.depends = depends +- self.blocks = blocks +- self.assignee = assignee +- self.comments = comments +- self.uid = uuid.uuid4().hex +- +- def to_json(self): +- ''' Returns a dictionary representation of the issue. +- +- ''' +- output = { +- 'id': self.id, +- 'title': self.title, +- 'content': self.content, +- 'status': self.status, +- 'date_created': self.date_created.strftime('%s'), +- 'user': self.user, +- 'private': self.private, +- 'tags': self.tags, +- 'depends': self.depends, +- 'blocks': self.blocks, +- 'assignee': self.assignee, +- 'comments': self.comments +- } +- +- return output +- +- @property +- def isa(self): +- return 'issue' +- +- +-class IssueComment(): +- ''' Represent a comment for an issue ''' +- +- def __init__( +- self, id, comment, date_created, +- user, parent=None, edited_on=None, editor=None): +- +- self.id = id +- self.comment = comment +- self.parent = parent +- self.date_created = date_created +- self.user = user +- self.edited_on = edited_on +- self.editor = editor +- +- def to_json(self): +- ''' Returns a dictionary representation of the issue. ''' +- +- output = { +- 'id': self.id, +- 'comment': self.comment, +- 'parent': self.parent, +- 'date_created': self.date_created.strftime('%s'), +- 'user': self.user, +- 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, +- 'editor': self.editor or None +- } +- +- return output +- +- +-class User(): +- ''' Represents a User ''' +- +- def __init__( +- self, name, emails, +- fullname=None): +- self.name = name +- self.fullname = fullname +- self.emails = emails +- +- def to_json(self): +- ''' Return a representation of the User in a dictionary. ''' +- +- output = { +- 'name': self.name, +- 'fullname': self.fullname, +- 'emails': self.emails +- } +- +- return output +diff --git a/pagure_importer/lib/repo.py b/pagure_importer/lib/repo.py +deleted file mode 100644 +index f46b31a..0000000 +--- a/pagure_importer/lib/repo.py ++++ /dev/null +@@ -1,70 +0,0 @@ +-# -*- coding: utf-8 -*- +- +-''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py +- by pingou@pingoured.fr +-''' +- +- +-import pygit2 +-import sys +- +- +-def get_pygit2_version(): +- ''' Return pygit2 version as a tuple of integers. +- This is needed for correct version comparison. +- ''' +- return tuple([int(i) for i in pygit2.__version__.split('.')]) +- +- +-class PagureRepo(pygit2.Repository): +- """ An utility class allowing to go around pygit2's inability to be +- stable. +- +- """ +- +- @staticmethod +- def push(remote, refname): +- """ Push the given reference to the specified remote. """ +- pygit2_version = get_pygit2_version() +- if pygit2_version >= (0, 22): +- remote.push([refname]) +- else: +- remote.push(refname) +- +- def pull(self, remote_name='origin', branch='master', force=False): +- ''' pull changes for the specified remote (defaults to origin). +- +- Code from MichaelBoselowitz at: +- https://github.com/MichaelBoselowitz/pygit2-examples/blob/ +- 68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58 +- licensed under the MIT license. +- ''' +- +- for remote in self.remotes: +- if remote.name == remote_name: +- remote.fetch() +- remote_master_id = self.lookup_reference( +- 'refs/remotes/origin/%s' % branch).target +- +- if force: +- repo_branch = self.lookup_reference( +- 'refs/heads/%s' % branch) +- repo_branch.set_target(remote_master_id) +- +- merge_result, _ = self.merge_analysis(remote_master_id) +- # Up to date, do nothing +- if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: +- return +- # We can just fastforward +- elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: +- self.checkout_tree(self.get(remote_master_id)) +- master_ref = self.lookup_reference( +- 'refs/heads/%s' % branch) +- master_ref.set_target(remote_master_id) +- self.head.set_target(remote_master_id) +- elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: +- sys.exit('Pulling remote changes leads to a conflict') +- else: +- print 'Unexpected merge result: %s' % ( +- pygit2.GIT_MERGE_ANALYSIS_NORMAL) +- raise AssertionError('Unknown merge analysis result') +diff --git a/pagure_importer/lib/sources/__init__.py b/pagure_importer/lib/sources/__init__.py +deleted file mode 100644 +index e69de29..0000000 +diff --git a/pagure_importer/lib/sources/importer_github.py b/pagure_importer/lib/sources/importer_github.py +deleted file mode 100644 +index 2859e99..0000000 +--- a/pagure_importer/lib/sources/importer_github.py ++++ /dev/null +@@ -1,134 +0,0 @@ +-from github import Github +-import pagure_importer +-import pagure_importer.lib +-from pagure_importer.lib import models +-from pagure_importer.lib import github_get_commentor_email +-from pagure_importer.lib.exceptions import GithubBadCredentials, GithubRepoNotFound +- +-class GithubImporter(): +- ''' Imports from Github using PyGithub and libpagure ''' +- def __init__( +- self, +- github_username, +- github_password, +- github_project_name): +- self.github_username = github_username +- self.github_password = github_password +- self.github_project_name = github_project_name +- self.github = Github(github_username, github_password) +- +- def import_issues(self, repo_path, repo_folder, status='all'): +- ''' Imports the issues on github for +- the given project +- ''' +- github_user = None +- try: +- github_user = self.github.get_user(self.github_username) +- except: +- raise GithubBadCredentials( +- 'Given github credentials are not correct') +- repo = self.github.get_repo(self.github_project_name) +- try: +- repo_name = repo.name +- except: +- raise GithubRepoNotFound( +- 'Repo not found, project name wrong') +- +- for github_issue in repo.get_issues(state=status): +- +- #title of the issue +- pagure_issue_title = github_issue.title +- +- #body of the issue +- if github_issue.body: +- pagure_issue_content = github_issue.body +- else: +- pagure_issue_content = '#No Description Provided' +- +- #Some details of a issue +- if github_issue.state != 'closed': +- pagure_issue_status = 'Open' +- else: +- pagure_issue_status = 'Fixed' +- +- pagure_issue_created_at = github_issue.created_at +- +- #Not sure how to deal with this atm +- pagure_issue_assignee = None +- +- if github_issue.labels: +- pagure_issue_tags = [i.name for i in github_issue.labels] +- else: +- pagure_issue_tags = [] +- +- +- #few things not supported by github +- pagure_issue_depends = [] +- pagure_issue_blocks = [] +- pagure_issue_is_private = False +- +- +- #User who created the issue +- pagure_issue_user = models.User( +- name=github_issue.user.login, +- fullname=github_issue.user.name, +- emails=[github_issue.user.email]) +- +- +- pagure_issue = models.Issue( +- id=None, +- title = pagure_issue_title, +- content = pagure_issue_content, +- status = pagure_issue_status, +- date_created = pagure_issue_created_at, +- user = pagure_issue_user.to_json(), +- private = pagure_issue_is_private, +- tags = pagure_issue_tags, +- depends = pagure_issue_depends, +- blocks = pagure_issue_blocks, +- assignee = pagure_issue_assignee) +- +- +- #comments on the issue +- comments = [] +- for comment in github_issue.get_comments(): +- +- comment_user = comment.user +- pagure_issue_comment_user_email = comment_user.email +- pagure_issue_comment_body = comment.body +- pagure_issue_comment_created_at = comment.created_at +- pagure_issue_comment_updated_at = comment.updated_at +- +- +- #No idea what to do with this right now +- #editor: not supported by github api +- pagure_issue_comment_parent = None +- pagure_issue_comment_editor = None +- +- #comment updated at +- pagure_issue_comment_edited_on = comment.updated_at +- +- #The User who commented +- pagure_issue_comment_user = models.User( +- name=comment_user.login, +- fullname=comment_user.name, +- emails=[comment_user.email] if comment_user.email \ +- else [github_get_commentor_email(comment_user.login)]) +- +- #Object to represent comment on an issue +- pagure_issue_comment = models.IssueComment( +- id=None, +- comment=pagure_issue_comment_body, +- parent=pagure_issue_comment_parent, +- date_created=pagure_issue_comment_created_at, +- user=pagure_issue_comment_user.to_json(), +- edited_on=pagure_issue_comment_edited_on, +- editor=pagure_issue_comment_editor) +- +- comments.append(pagure_issue_comment.to_json()) +- +- #add all the comments to the issue object +- pagure_issue.comments = comments +- +- #update the local git repo +- pagure_importer.lib.git.update_git(pagure_issue, repo_path, repo_folder) +diff --git a/pagure_importer/lib/sources/importer_trac.py b/pagure_importer/lib/sources/importer_trac.py +deleted file mode 100644 +index f69a3ce..0000000 +--- a/pagure_importer/lib/sources/importer_trac.py ++++ /dev/null +@@ -1,132 +0,0 @@ +-from xmlrpclib import ServerProxy +-import pagure_importer +-import pagure_importer.lib +-from pagure_importer.lib.fas import FASclient +-from pagure_importer.lib import trac +- +- +-class TracImporter(): +- '''Pagure importer for trac instance''' +- +- def __init__(self, trac_project_url): +- self.tracclient = ServerProxy(trac_project_url + '/rpc') +- self.fasclient = FASclient('user', 'password', +- 'https://admin.fedoraproject.org/accounts') +- +- def _find_fas_user(self, user): +- person = self.fasclient.person_by_username(user) +- human_name = person['human_name'] +- email = person['email'] +- pagure_user = models.User( +- name=user, +- fullname=human_name, +- emails=[email]) +- return pagure_user +- +- def _get_ticket_tags(self, trac_ticket): +- return [] +- +- def _get_ticket_status(self, trac_ticket): +- ''' Converts Trac ticket status +- to Pagure issue status''' +- +- if trac_ticket['status'] != 'closed': +- ticket_status = 'Open' +- else: +- ticket_status = 'Fixed' +- return ticket_status +- +- def _populate_comments(self, trac_comments): +- comments = [] +- for comment in trac_comments: +- if comment[2] == 'comment' and comment[4] != '': +- comment_user = comment[1] +- pagure_issue_comment_user_email = None +- pagure_issue_comment_body = comment[4] +- pagure_issue_comment_created_at = datetime.strptime( +- comment[0].value, "%Y%m%dT%H:%M:%S") +- pagure_issue_comment_updated_at = None +- +- # No idea what to do with this right now +- # editor: not supported by github api +- pagure_issue_comment_parent = None +- pagure_issue_comment_editor = None +- +- # comment updated at +- pagure_issue_comment_edited_on = None +- +- # The User who commented +- pagure_issue_comment_user = self._find_fas_user(comment[1]) +- +- # Object to represent comment on an issue +- pagure_issue_comment = models.IssueComment( +- id=None, +- comment=pagure_issue_comment_body, +- parent=pagure_issue_comment_parent, +- date_created=pagure_issue_comment_created_at, +- user=pagure_issue_comment_user.to_json(), +- edited_on=pagure_issue_comment_edited_on, +- editor=pagure_issue_comment_editor) +- +- comments.append(pagure_issue_comment.to_json()) +- return comments +- +- def _populate_issue(self, ticket_id): +- trac_ticket = self.trac.ticket.get(ticket_id)[3] +- pagure_issue_title = trac_ticket['summary'] +- pagure_issue_content = trac_ticket['description'] +- +- if pagure_issue_content == '': +- pagure_issue_content = '#No Description Provided' +- +- pagure_issue_status = self._get_ticket_status(trac_ticket) +- +- pagure_issue_created_at = datetime.strptime( +- self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") +- +- pagure_issue_assignee = trac_ticket['owner'] +- +- pagure_issue_tags = self._get_ticket_tags(trac_ticket) +- +- pagure_issue_depends = [] +- pagure_issue_blocks = [] +- pagure_issue_is_private = False +- +- pagure_issue_user = self._find_fas_user(trac_ticket['reporter']) +- +- pagure_issue = models.Issue( +- id=None, +- title=pagure_issue_title, +- content=pagure_issue_content, +- status=pagure_issue_status, +- date_created=pagure_issue_created_at, +- user=pagure_issue_user.to_json(), +- private=pagure_issue_is_private, +- tags=pagure_issue_tags, +- depends=pagure_issue_depends, +- blocks=pagure_issue_blocks, +- assignee=pagure_issue_assignee) +- return pagure_issue +- +- def import_issues(self, repo_path, repo_folder, +- trac_query='max=0&order=id'): +- '''Import issues from trac instance using xmlrpc API''' +- tickets_id = self.tracclient.ticket.query(trac_query) +- +- for ticket_id in tickets_id: +- +- pagure_issue = trac.populate_issue(self.tracclient, +- self.fasclient, ticket_id) +- +- pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) +- comments = trac.populate_comments(self.fasclient, +- pagure_issue_comments) +- +- # add all the comments to the issue object +- pagure_issue.comments = comments +- +- # update the local git repo +- print 'Update repo with issue :' + str(ticket_id) +- pagure_importer.lib.git.update_git(pagure_issue, +- repo_path, +- repo_folder) +diff --git a/pagure_importer/lib/trac.py b/pagure_importer/lib/trac.py +deleted file mode 100644 +index fb3d062..0000000 +--- a/pagure_importer/lib/trac.py ++++ /dev/null +@@ -1,90 +0,0 @@ +-from pagure_importer.lib.models import IssueComment, Issue +-from datetime import datetime +- +- +-def get_ticket_tags(trac_ticket): +- return [] +- +- +-def get_ticket_status(trac_ticket): +- ''' Converts Trac ticket status +- to Pagure issue status''' +- +- if trac_ticket['status'] != 'closed': +- ticket_status = 'Open' +- else: +- ticket_status = 'Fixed' +- return ticket_status +- +- +-def populate_comments(fasclient, trac_comments): +- comments = [] +- for comment in trac_comments: +- if comment[2] == 'comment' and comment[4] != '': +- comment_user = comment[1] +- pagure_issue_comment_user_email = None +- pagure_issue_comment_body = comment[4] +- pagure_issue_comment_created_at = datetime.strptime( +- comment[0].value, "%Y%m%dT%H:%M:%S") +- pagure_issue_comment_updated_at = None +- +- # No idea what to do with this right now +- # editor: not supported by github api +- pagure_issue_comment_parent = None +- pagure_issue_comment_editor = None +- +- # comment updated at +- pagure_issue_comment_edited_on = None +- +- # The User who commented +- pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) +- +- # Object to represent comment on an issue +- pagure_issue_comment = IssueComment( +- id=None, +- comment=pagure_issue_comment_body, +- parent=pagure_issue_comment_parent, +- date_created=pagure_issue_comment_created_at, +- user=pagure_issue_comment_user.to_json(), +- edited_on=pagure_issue_comment_edited_on, +- editor=pagure_issue_comment_editor) +- +- comments.append(pagure_issue_comment.to_json()) +- return comments +- +- +-def populate_issue(trac, fasclient, ticket_id): +- trac_ticket = trac.ticket.get(ticket_id)[3] +- pagure_issue_title = trac_ticket['summary'] +- pagure_issue_content = trac_ticket['description'] +- +- if pagure_issue_content == '': +- pagure_issue_content = '#No Description Provided' +- +- pagure_issue_status = get_ticket_status(trac_ticket) +- +- pagure_issue_created_at = datetime.strptime( +- trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") +- +- pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) +- +- pagure_issue_tags = get_ticket_tags(trac_ticket) +- +- pagure_issue_depends = [] +- pagure_issue_blocks = [] +- pagure_issue_is_private = False +- +- pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) +- pagure_issue = Issue( +- id=ticket_id, +- title=pagure_issue_title, +- content=pagure_issue_content, +- status=pagure_issue_status, +- date_created=pagure_issue_created_at, +- user=pagure_issue_user.to_json(), +- private=pagure_issue_is_private, +- tags=pagure_issue_tags, +- depends=pagure_issue_depends, +- blocks=pagure_issue_blocks, +- assignee=pagure_issue_assignee.to_json()) +- return pagure_issue +diff --git a/pagure_importer/run.py b/pagure_importer/run.py +deleted file mode 100644 +index d327228..0000000 +--- a/pagure_importer/run.py ++++ /dev/null +@@ -1,63 +0,0 @@ +-#!/usr/bin/env python +-import getpass +-from forms import form_github_issues +-from settings import IMPORT_SOURCES, IMPORT_OPTIONS, REPO_NAME, REPO_PATH +-import pagure_importer +-import pagure_importer.lib +-import pagure_importer.lib.sources +-from pagure_importer.lib.sources.importer_github import GithubImporter +-from pagure_importer.lib.sources.importer_trac import TracImporter +-from pagure_importer.lib import generate_json_for_github_contributors, \ +- generate_json_for_github_issue_commentors, \ +- assemble_github_contributors_commentors +- +- +-def github_handler(item): +- if item.lower() == 'issues': +- github_username, github_password, github_project_name = form_github_issues() +- gen_json = raw_input('Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') +- if gen_json == 'n': +- github_importer = GithubImporter( +- github_username=github_username, +- github_password=github_password, +- github_project_name=github_project_name) +- github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) +- else: +- generate_json_for_github_contributors( +- github_username, +- github_password, +- github_project_name) +- generate_json_for_github_issue_commentors( +- github_username, +- github_password, +- github_project_name) +- assemble_github_contributors_commentors() +- return +- +- +-def trac_handler(item, fedora=False): +- if item.lower() == 'issues': +- trac_url = raw_input('Enter the trac project url: ') +- trac_importer = TracImporter(trac_url) +- trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) +- +- +-def main(): +- source = raw_input('Enter source from where you want to import: ') +- if source.lower() not in IMPORT_SOURCES: +- print 'Source location not supported' +- return +- +- item = raw_input('Enter the item to be imported: ') +- if item.lower() not in IMPORT_OPTIONS[source]: +- print 'Item import not supported' +- return +- +- if source.lower() == 'github': +- github_handler(item) +- elif source.lower() == 'fedorahosted': +- trac_handler(item, fedora=True) +- return +- +-if __name__ == '__main__': +- main() +diff --git a/pagure_importer/settings.py b/pagure_importer/settings.py +deleted file mode 100644 +index 077ea40..0000000 +--- a/pagure_importer/settings.py ++++ /dev/null +@@ -1,7 +0,0 @@ +-import os +- +-IMPORT_SOURCES = ['github', 'fedorahosted'] +-IMPORT_OPTIONS = {'github': ['issues'], 'fedorahosted': ['issues']} +- +-REPO_NAME = os.environ.get('REPO_NAME', None) #this has to be a bare repo +-REPO_PATH = os.environ.get('REPO_PATH', None) #the parent of the git directory +diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py +new file mode 100644 +index 0000000..05ccdb8 +--- /dev/null ++++ b/pagure_importer/utils/__init__.py +@@ -0,0 +1,168 @@ ++import git ++import models ++ ++import csv ++import os ++import getpass ++import requests ++import json ++from github import Github ++from requests.auth import HTTPBasicAuth ++ ++from pagure_importer.utils.exceptions import FileNotFound, EmailNotFound ++ ++def generate_json_for_github_contributors(github_username, github_password, \ ++ github_project_name): ++ ''' Creates a file containing a list of dicts containing the username and ++ emails of the contributors in the given github project ++ ''' ++ ++ github_obj = Github(github_username, github_password) ++ project = github_obj.get_repo(github_project_name) ++ commits_url = project.commits_url.replace('{/sha}', '') ++ ++ page = 0 ++ contributors = [] ++ while True: ++ page += 1 ++ payload = {'page': page } ++ data_ = json.loads(requests.get(commits_url, params=payload, ++ auth=HTTPBasicAuth(github_username, github_password)).text) ++ ++ if not data_: ++ break ++ ++ for data in data_: ++ try: ++ contributor = data['commit']['committer'] ++ contributor_email = contributor['email'] ++ contributor_fullname = contributor['name'] ++ contributor_name = data['committer']['login'] ++ except TypeError: ++ print 'Maybe one of the contributors is dropped because of lack of details' ++ continue ++ ++ json_data = { ++ 'name': contributor_name, ++ 'fullname': contributor_fullname, ++ 'emails': [contributor_email] ++ } ++ ++ present = False ++ for i in contributors: ++ if i == json_data: ++ present = True ++ break ++ ++ if not present: ++ print 'contributor added: ', contributor_name ++ contributors.append(json_data) ++ ++ with open('contributors.json', 'w') as f: ++ f.write(json.dumps(contributors)) ++ ++ return ++ ++ ++def generate_json_for_github_issue_commentors(github_username, github_password, \ ++ github_project_name): ++ ''' Will create a json file containing details of all the user ++ who have commented on any issue in the given project ++ ''' ++ ++ github_obj = Github(github_username, github_password) ++ project = github_obj.get_repo(github_project_name) ++ issue_comment_url = project.issue_comment_url.replace('{/number}', '') ++ ++ page = 0 ++ issue_commentors = [] ++ while True: ++ page += 1 ++ payload = {'page': page } ++ data_ = json.loads(requests.get(issue_comment_url, params=payload, ++ auth=HTTPBasicAuth(github_username, github_password)).text) ++ ++ if not data_: ++ break ++ ++ for data in data_: ++ try: ++ commentor = data['user']['login'] ++ except TypeError: ++ print 'Maybe one of the issue commentors have been dropped because of lack of details' ++ continue ++ ++ present = False ++ for i in issue_commentors: ++ if i == commentor: ++ present = True ++ break ++ ++ if not present: ++ print 'commentor added: ', commentor ++ issue_commentors.append(commentor) ++ ++ with open('issue_commentors.json', 'w') as f: ++ f.write(json.dumps(issue_commentors)) ++ return ++ ++ ++def assemble_github_contributors_commentors(): ++ ''' It uses the files: issue_commentors.json and contributors.json ++ Assembles and creates a file: assembled_commentors.csv ++ To use: just fill the empty blocks under emails column''' ++ ++ with open('issue_commentors.json', 'r') as ic: ++ issue_names = json.load(ic) ++ ++ with open('contributors.json', 'r') as c: ++ contributors = json.load(c) ++ ++ names = [] ++ for i in issue_names: ++ found = False ++ for j in contributors: ++ if j.get('name', None) == i: ++ j['emails'] = j.get('emails')[0] ++ names.append(j) ++ found = True ++ ++ if not found: ++ d = {'name': i, 'fullname': None, 'emails': None} ++ names.append(d) ++ ++ with open('assembled_commentors.csv', 'w') as ac: ++ field_names = ['name', 'fullname', 'emails'] ++ writer = csv.DictWriter(ac, fieldnames=field_names) ++ ++ writer.writeheader() ++ for name in names: ++ writer.writerow(name) ++ ++ ++def github_get_commentor_email(name): ++ ''' Will return the issue commentor email as given in the ++ assembled_commentors.csv file ++ ''' ++ ++ if not os.path.exists('assembled_commentors.csv'): ++ raise FileNotFound('The assembled_commentors.json file must be present \ ++ Rerun the program and choose to generate the json files') ++ ++ data = [] ++ with open('assembled_commentors.csv') as ac: ++ reader = csv.DictReader(ac) ++ for row in reader: ++ data.append(dict( \ ++ (('name', row['name']), \ ++ ('fullname', row['fullname']), \ ++ ('emails', row['emails'])))) ++ ++ ++ for i in data: ++ if i.get('name', None) == name: ++ if i['emails']: ++ return str(i['emails']) ++ else: ++ raise EmailNotFound('You need to fill out all the emails of the \ ++ issue commentors') +diff --git a/pagure_importer/utils/exceptions.py b/pagure_importer/utils/exceptions.py +new file mode 100644 +index 0000000..a5ab5fe +--- /dev/null ++++ b/pagure_importer/utils/exceptions.py +@@ -0,0 +1,22 @@ ++class GithubBadCredentials(Exception): ++ ''' Raised when username/password for github is wrong ''' ++ def __init__(self, msg): ++ self.msg = msg ++ ++ ++class GithubRepoNotFound(Exception): ++ ''' Raised when the repo is not found for the user ''' ++ def __init__(self, msg): ++ self.msg = msg ++ ++ ++class FileNotFound(Exception): ++ ''' Raised when a certain file is not found ''' ++ def __init__(self, msg): ++ self.msg = msg ++ ++ ++class EmailNotFound(Exception): ++ ''' Raised when email is not found ''' ++ def __init__(self, msg): ++ self.msg = msg +diff --git a/pagure_importer/utils/fas.py b/pagure_importer/utils/fas.py +new file mode 100644 +index 0000000..f62b5f5 +--- /dev/null ++++ b/pagure_importer/utils/fas.py +@@ -0,0 +1,23 @@ ++from fedora.client.fas2 import AccountSystem ++from pagure_importer.utils.models import User ++ ++ ++class FASclient (): ++ def __init__(self, fas_username, fas_password, fas_url): ++ self.fasclient = AccountSystem(fas_url, username=fas_username, ++ password=fas_password) ++ ++ anonymous = User(name='', fullname='', emails=[]) ++ self.fasuser = {'': anonymous} ++ ++ def find_fas_user(self, user): ++ ++ if user not in self.fasuser.keys(): ++ person = self.fasclient.person_by_username(user) ++ if not person: ++ return self.fasuser[''] ++ ++ self.fasuser[user] = User(name=user, ++ fullname=person['human_name'], ++ emails=[person['email']]) ++ return self.fasuser[user] +diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py +new file mode 100644 +index 0000000..33eba48 +--- /dev/null ++++ b/pagure_importer/utils/git.py +@@ -0,0 +1,98 @@ ++''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py ++ by pingou@pingoured.fr ++''' ++ ++import shutil ++import os ++import pygit2 ++import tempfile ++import json ++ ++from repo import * ++ ++def update_git(obj, repo_path, repo_folder): ++ """ Update the given issue in its git. ++ This method forks the provided repo, add/edit the issue whose file name ++ is defined by the uid field of the issue and if there are additions/ ++ changes commit them and push them back to the original repo. ++ """ ++ ++ if not repo_folder: ++ return ++ ++ # Get the fork ++ repopath = os.path.join(repo_folder, repo_path) ++ ++ # Clone the repo into a temp folder ++ newpath = tempfile.mkdtemp(prefix='pagure-') ++ new_repo = pygit2.clone_repository(repopath, newpath) ++ ++ file_path = os.path.join(newpath, obj.uid) ++ ++ # Get the current index ++ index = new_repo.index ++ ++ # Are we adding files ++ added = False ++ if not os.path.exists(file_path): ++ added = True ++ ++ # Write down what changed ++ with open(file_path, 'w') as stream: ++ stream.write(json.dumps( ++ obj.to_json(), sort_keys=True, indent=4, ++ separators=(',', ': '))) ++ ++ # Retrieve the list of files that changed ++ diff = new_repo.diff() ++ files = [] ++ for p in diff: ++ if hasattr(p, 'new_file_path'): ++ files.append(p.new_file_path) ++ elif hasattr(p, 'delta'): ++ files.append(p.delta.new_file.path) ++ ++ # Add the changes to the index ++ if added: ++ index.add(obj.uid) ++ for filename in files: ++ index.add(filename) ++ ++ # If not change, return ++ if not files and not added: ++ shutil.rmtree(newpath) ++ return ++ ++ # See if there is a parent to this commit ++ parent = None ++ try: ++ parent = new_repo.head.get_object().oid ++ except pygit2.GitError: ++ pass ++ ++ parents = [] ++ if parent: ++ parents.append(parent) ++ ++ # Author/commiter will always be this one ++ author = pygit2.Signature(name='pagure', email='pagure') ++ ++ # Actually commit ++ new_repo.create_commit( ++ 'refs/heads/master', ++ author, ++ author, ++ 'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title), ++ new_repo.index.write_tree(), ++ parents) ++ index.write() ++ ++ # Push to origin ++ ori_remote = new_repo.remotes[0] ++ master_ref = new_repo.lookup_reference('HEAD').resolve() ++ refname = '%s:%s' % (master_ref.name, master_ref.name) ++ ++ PagureRepo.push(ori_remote, refname) ++ ++ # Remove the clone ++ shutil.rmtree(newpath) +diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py +new file mode 100644 +index 0000000..cc61370 +--- /dev/null ++++ b/pagure_importer/utils/importer_github.py +@@ -0,0 +1,138 @@ ++from github import Github ++ ++from pagure_importer.utils import models ++from pagure_importer.utils import github_get_commentor_email ++from pagure_importer.utils.git import update_git ++from pagure_importer.utils.exceptions import ( ++ GithubBadCredentials, ++ GithubRepoNotFound ++) ++ ++ ++class GithubImporter(): ++ ''' Imports from Github using PyGithub and libpagure ''' ++ def __init__( ++ self, ++ github_username, ++ github_password, ++ github_project_name): ++ self.github_username = github_username ++ self.github_password = github_password ++ self.github_project_name = github_project_name ++ self.github = Github(github_username, github_password) ++ ++ def import_issues(self, repo_path, repo_folder, status='all'): ++ ''' Imports the issues on github for ++ the given project ++ ''' ++ github_user = None ++ try: ++ github_user = self.github.get_user(self.github_username) ++ except: ++ raise GithubBadCredentials( ++ 'Given github credentials are not correct') ++ repo = self.github.get_repo(self.github_project_name) ++ try: ++ repo_name = repo.name ++ except: ++ raise GithubRepoNotFound( ++ 'Repo not found, project name wrong') ++ ++ for github_issue in repo.get_issues(state=status): ++ ++ # title of the issue ++ pagure_issue_title = github_issue.title ++ ++ # body of the issue ++ if github_issue.body: ++ pagure_issue_content = github_issue.body ++ else: ++ pagure_issue_content = '#No Description Provided' ++ ++ # Some details of a issue ++ if github_issue.state != 'closed': ++ pagure_issue_status = 'Open' ++ else: ++ pagure_issue_status = 'Fixed' ++ ++ pagure_issue_created_at = github_issue.created_at ++ ++ # Not sure how to deal with this atm ++ pagure_issue_assignee = None ++ ++ if github_issue.labels: ++ pagure_issue_tags = [i.name for i in github_issue.labels] ++ else: ++ pagure_issue_tags = [] ++ ++ ++ # few things not supported by github ++ pagure_issue_depends = [] ++ pagure_issue_blocks = [] ++ pagure_issue_is_private = False ++ ++ ++ # User who created the issue ++ pagure_issue_user = models.User( ++ name=github_issue.user.login, ++ fullname=github_issue.user.name, ++ emails=[github_issue.user.email]) ++ ++ ++ pagure_issue = models.Issue( ++ id=None, ++ title = pagure_issue_title, ++ content = pagure_issue_content, ++ status = pagure_issue_status, ++ date_created = pagure_issue_created_at, ++ user = pagure_issue_user.to_json(), ++ private = pagure_issue_is_private, ++ tags = pagure_issue_tags, ++ depends = pagure_issue_depends, ++ blocks = pagure_issue_blocks, ++ assignee = pagure_issue_assignee) ++ ++ ++ # comments on the issue ++ comments = [] ++ for comment in github_issue.get_comments(): ++ ++ comment_user = comment.user ++ pagure_issue_comment_user_email = comment_user.email ++ pagure_issue_comment_body = comment.body ++ pagure_issue_comment_created_at = comment.created_at ++ pagure_issue_comment_updated_at = comment.updated_at ++ ++ ++ # No idea what to do with this right now ++ # editor: not supported by github api ++ pagure_issue_comment_parent = None ++ pagure_issue_comment_editor = None ++ ++ # comment updated at ++ pagure_issue_comment_edited_on = comment.updated_at ++ ++ # The User who commented ++ pagure_issue_comment_user = models.User( ++ name=comment_user.login, ++ fullname=comment_user.name, ++ emails=[comment_user.email] if comment_user.email \ ++ else [github_get_commentor_email(comment_user.login)]) ++ ++ # Object to represent comment on an issue ++ pagure_issue_comment = models.IssueComment( ++ id=None, ++ comment=pagure_issue_comment_body, ++ parent=pagure_issue_comment_parent, ++ date_created=pagure_issue_comment_created_at, ++ user=pagure_issue_comment_user.to_json(), ++ edited_on=pagure_issue_comment_edited_on, ++ editor=pagure_issue_comment_editor) ++ ++ comments.append(pagure_issue_comment.to_json()) ++ ++ # add all the comments to the issue object ++ pagure_issue.comments = comments ++ ++ # update the local git repo ++ update_git(pagure_issue, repo_path, repo_folder) +diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py +new file mode 100644 +index 0000000..50f20b0 +--- /dev/null ++++ b/pagure_importer/utils/importer_trac.py +@@ -0,0 +1,35 @@ ++from xmlrpclib import ServerProxy ++from pagure_importer.utils.git import update_git ++from pagure_importer.utils import trac ++ ++ ++class TracImporter(): ++ '''Pagure importer for trac instance''' ++ ++ def __init__(self, trac_project_url, fasclient=None): ++ self.tracclient = ServerProxy(trac_project_url + '/rpc') ++ if fasclient: ++ self.fasclient = fasclient ++ ++ def import_issues(self, repo_path, repo_folder, ++ trac_query='max=0&order=id'): ++ '''Import issues from trac instance using xmlrpc API''' ++ tickets_id = self.tracclient.ticket.query(trac_query) ++ ++ for ticket_id in tickets_id: ++ ++ pagure_issue = trac.populate_issue(self.tracclient, ++ self.fasclient, ticket_id) ++ ++ pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) ++ comments = trac.populate_comments(self.fasclient, ++ pagure_issue_comments) ++ ++ # add all the comments to the issue object ++ pagure_issue.comments = comments ++ ++ # update the local git repo ++ print 'Update repo with issue :' + str(ticket_id) ++ update_git(pagure_issue, ++ repo_path, ++ repo_folder) +diff --git a/pagure_importer/utils/models.py b/pagure_importer/utils/models.py +new file mode 100644 +index 0000000..3aa3d83 +--- /dev/null ++++ b/pagure_importer/utils/models.py +@@ -0,0 +1,105 @@ ++# -*- coding: utf-8 -*- ++ ++import datetime ++import json ++import uuid ++ ++class Issue(): ++ ''' Represents an Issue ''' ++ ++ def __init__( ++ self, id, title, content, ++ status, date_created, user, private, tags, ++ depends, blocks, assignee, comments=None): ++ ++ self.id = id ++ self.title = title ++ self.content = content ++ self.status = status ++ self.date_created = date_created ++ self.user = user ++ self.private = private ++ self.tags = tags ++ self.depends = depends ++ self.blocks = blocks ++ self.assignee = assignee ++ self.comments = comments ++ self.uid = uuid.uuid4().hex ++ ++ def to_json(self): ++ ''' Returns a dictionary representation of the issue. ++ ++ ''' ++ output = { ++ 'id': self.id, ++ 'title': self.title, ++ 'content': self.content, ++ 'status': self.status, ++ 'date_created': self.date_created.strftime('%s'), ++ 'user': self.user, ++ 'private': self.private, ++ 'tags': self.tags, ++ 'depends': self.depends, ++ 'blocks': self.blocks, ++ 'assignee': self.assignee, ++ 'comments': self.comments ++ } ++ ++ return output ++ ++ @property ++ def isa(self): ++ return 'issue' ++ ++ ++class IssueComment(): ++ ''' Represent a comment for an issue ''' ++ ++ def __init__( ++ self, id, comment, date_created, ++ user, parent=None, edited_on=None, editor=None): ++ ++ self.id = id ++ self.comment = comment ++ self.parent = parent ++ self.date_created = date_created ++ self.user = user ++ self.edited_on = edited_on ++ self.editor = editor ++ ++ def to_json(self): ++ ''' Returns a dictionary representation of the issue. ''' ++ ++ output = { ++ 'id': self.id, ++ 'comment': self.comment, ++ 'parent': self.parent, ++ 'date_created': self.date_created.strftime('%s'), ++ 'user': self.user, ++ 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, ++ 'editor': self.editor or None ++ } ++ ++ return output ++ ++ ++class User(): ++ ''' Represents a User ''' ++ ++ def __init__( ++ self, name, emails, ++ fullname=None): ++ self.name = name ++ self.fullname = fullname ++ self.emails = emails ++ ++ def to_json(self): ++ ''' Return a representation of the User in a dictionary. ''' ++ ++ output = { ++ 'name': self.name, ++ 'fullname': self.fullname, ++ 'emails': self.emails ++ } ++ ++ return output +diff --git a/pagure_importer/utils/repo.py b/pagure_importer/utils/repo.py +new file mode 100644 +index 0000000..f46b31a +--- /dev/null ++++ b/pagure_importer/utils/repo.py +@@ -0,0 +1,70 @@ ++# -*- coding: utf-8 -*- ++ ++''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py ++ by pingou@pingoured.fr ++''' ++ ++ ++import pygit2 ++import sys ++ ++ ++def get_pygit2_version(): ++ ''' Return pygit2 version as a tuple of integers. ++ This is needed for correct version comparison. ++ ''' ++ return tuple([int(i) for i in pygit2.__version__.split('.')]) ++ ++ ++class PagureRepo(pygit2.Repository): ++ """ An utility class allowing to go around pygit2's inability to be ++ stable. ++ ++ """ ++ ++ @staticmethod ++ def push(remote, refname): ++ """ Push the given reference to the specified remote. """ ++ pygit2_version = get_pygit2_version() ++ if pygit2_version >= (0, 22): ++ remote.push([refname]) ++ else: ++ remote.push(refname) ++ ++ def pull(self, remote_name='origin', branch='master', force=False): ++ ''' pull changes for the specified remote (defaults to origin). ++ ++ Code from MichaelBoselowitz at: ++ https://github.com/MichaelBoselowitz/pygit2-examples/blob/ ++ 68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58 ++ licensed under the MIT license. ++ ''' ++ ++ for remote in self.remotes: ++ if remote.name == remote_name: ++ remote.fetch() ++ remote_master_id = self.lookup_reference( ++ 'refs/remotes/origin/%s' % branch).target ++ ++ if force: ++ repo_branch = self.lookup_reference( ++ 'refs/heads/%s' % branch) ++ repo_branch.set_target(remote_master_id) ++ ++ merge_result, _ = self.merge_analysis(remote_master_id) ++ # Up to date, do nothing ++ if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: ++ return ++ # We can just fastforward ++ elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: ++ self.checkout_tree(self.get(remote_master_id)) ++ master_ref = self.lookup_reference( ++ 'refs/heads/%s' % branch) ++ master_ref.set_target(remote_master_id) ++ self.head.set_target(remote_master_id) ++ elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: ++ sys.exit('Pulling remote changes leads to a conflict') ++ else: ++ print 'Unexpected merge result: %s' % ( ++ pygit2.GIT_MERGE_ANALYSIS_NORMAL) ++ raise AssertionError('Unknown merge analysis result') +diff --git a/pagure_importer/utils/trac.py b/pagure_importer/utils/trac.py +new file mode 100644 +index 0000000..d56a123 +--- /dev/null ++++ b/pagure_importer/utils/trac.py +@@ -0,0 +1,90 @@ ++from pagure_importer.utils.models import IssueComment, Issue ++from datetime import datetime ++ ++ ++def get_ticket_tags(trac_ticket): ++ return [] ++ ++ ++def get_ticket_status(trac_ticket): ++ ''' Converts Trac ticket status ++ to Pagure issue status''' ++ ++ if trac_ticket['status'] != 'closed': ++ ticket_status = 'Open' ++ else: ++ ticket_status = 'Fixed' ++ return ticket_status ++ ++ ++def populate_comments(fasclient, trac_comments): ++ comments = [] ++ for comment in trac_comments: ++ if comment[2] == 'comment' and comment[4] != '': ++ comment_user = comment[1] ++ pagure_issue_comment_user_email = None ++ pagure_issue_comment_body = comment[4] ++ pagure_issue_comment_created_at = datetime.strptime( ++ comment[0].value, "%Y%m%dT%H:%M:%S") ++ pagure_issue_comment_updated_at = None ++ ++ # No idea what to do with this right now ++ # editor: not supported by github api ++ pagure_issue_comment_parent = None ++ pagure_issue_comment_editor = None ++ ++ # comment updated at ++ pagure_issue_comment_edited_on = None ++ ++ # The User who commented ++ pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) ++ ++ # Object to represent comment on an issue ++ pagure_issue_comment = IssueComment( ++ id=None, ++ comment=pagure_issue_comment_body, ++ parent=pagure_issue_comment_parent, ++ date_created=pagure_issue_comment_created_at, ++ user=pagure_issue_comment_user.to_json(), ++ edited_on=pagure_issue_comment_edited_on, ++ editor=pagure_issue_comment_editor) ++ ++ comments.append(pagure_issue_comment.to_json()) ++ return comments ++ ++ ++def populate_issue(trac, fasclient, ticket_id): ++ trac_ticket = trac.ticket.get(ticket_id)[3] ++ pagure_issue_title = trac_ticket['summary'] ++ pagure_issue_content = trac_ticket['description'] ++ ++ if pagure_issue_content == '': ++ pagure_issue_content = '#No Description Provided' ++ ++ pagure_issue_status = get_ticket_status(trac_ticket) ++ ++ pagure_issue_created_at = datetime.strptime( ++ trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") ++ ++ pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) ++ ++ pagure_issue_tags = get_ticket_tags(trac_ticket) ++ ++ pagure_issue_depends = [] ++ pagure_issue_blocks = [] ++ pagure_issue_is_private = False ++ ++ pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) ++ pagure_issue = Issue( ++ id=ticket_id, ++ title=pagure_issue_title, ++ content=pagure_issue_content, ++ status=pagure_issue_status, ++ date_created=pagure_issue_created_at, ++ user=pagure_issue_user.to_json(), ++ private=pagure_issue_is_private, ++ tags=pagure_issue_tags, ++ depends=pagure_issue_depends, ++ blocks=pagure_issue_blocks, ++ assignee=pagure_issue_assignee.to_json()) ++ return pagure_issue +diff --git a/setup.py b/setup.py +index e031405..65fa0d5 100644 +--- a/setup.py ++++ b/setup.py +@@ -32,7 +32,7 @@ setup( + license='GNU General Public License v2.0', + entry_points={ + 'console_scripts': [ +- 'pgimport = pagure_importer.run:main' ++ 'pgimport = pagure_importer.app:app' + ], + }, + include_package_data=True, diff --git a/pagure_importer/__init__.py b/pagure_importer/__init__.py index fbab797..8b13789 100644 --- a/pagure_importer/__init__.py +++ b/pagure_importer/__init__.py @@ -1,2 +1 @@ -import lib -import settings + diff --git a/pagure_importer/app.py b/pagure_importer/app.py new file mode 100644 index 0000000..6dc3277 --- /dev/null +++ b/pagure_importer/app.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +import click +import os + +REPO_NAME = os.environ.get('REPO_NAME', None) # this has to be a bare repo +REPO_PATH = os.environ.get('REPO_PATH', None) # the parent of the git directory + + +@click.group() +def app(): + pass + +__all__ = [ + 'app', +] + +# from .commands import github +from .commands import fedorahosted +from .commands import github + +if __name__ == '__main__': + app() diff --git a/pagure_importer/commands/__init__.py b/pagure_importer/commands/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pagure_importer/commands/__init__.py diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py new file mode 100644 index 0000000..9ca8a53 --- /dev/null +++ b/pagure_importer/commands/fedorahosted.py @@ -0,0 +1,15 @@ +import click +import getpass +from pagure_importer.app import app, REPO_NAME, REPO_PATH +from pagure_importer.utils import importer_trac +from pagure_importer.utils.fas import FASclient + +@app.command() +@click.argument('project_url') +def fedorahosted(project_url): + fas_username = raw_input('Enter you FAS Username: ') + fas_password = getpass.getpass('Enter your FAS password: ') + fasclient = FASclient(fas_username, fas_password, + 'https://admin.fedoraproject.org/accounts') + trac_importer = importer_trac.TracImporter(project_url, fasclient) + trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py new file mode 100644 index 0000000..26ae4fa --- /dev/null +++ b/pagure_importer/commands/github.py @@ -0,0 +1,40 @@ +import click +import getpass + +from pagure_importer.app import app, REPO_NAME, REPO_PATH +from pagure_importer.utils.importer_github import GithubImporter +from pagure_importer.utils import ( + generate_json_for_github_contributors, + generate_json_for_github_issue_commentors, + assemble_github_contributors_commentors +) + + +def form_github_issues(): + github_username = raw_input('Enter you Github Username: ') + github_password = getpass.getpass('Enter your github password: ') + github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') + return (github_username, github_password, github_project_name) + +@app.command() +def github(): + github_username, github_password, github_project_name = form_github_issues() + gen_json = raw_input( + 'Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') + if gen_json == 'n': + github_importer = GithubImporter( + github_username=github_username, + github_password=github_password, + github_project_name=github_project_name) + github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) + else: + generate_json_for_github_contributors( + github_username, + github_password, + github_project_name) + generate_json_for_github_issue_commentors( + github_username, + github_password, + github_project_name) + assemble_github_contributors_commentors() + return diff --git a/pagure_importer/forms.py b/pagure_importer/forms.py deleted file mode 100644 index a111114..0000000 --- a/pagure_importer/forms.py +++ /dev/null @@ -1,10 +0,0 @@ -import getpass - -from lib.sources.importer_github import GithubImporter -from settings import REPO_PATH, REPO_NAME - -def form_github_issues(): - github_username = raw_input('Enter you Github Username: ') - github_password = getpass.getpass('Enter your github password: ') - github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') - return (github_username, github_password, github_project_name) diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py deleted file mode 100644 index 9a64e3b..0000000 --- a/pagure_importer/lib/__init__.py +++ /dev/null @@ -1,170 +0,0 @@ -import git -import models - -import csv -import os -import getpass -import requests -import json -from github import Github -from requests.auth import HTTPBasicAuth -import pagure_importer -import pagure_importer.lib -from pagure_importer.lib.exceptions import FileNotFound, EmailNotFound - -def generate_json_for_github_contributors(github_username, github_password, \ - github_project_name): - ''' Creates a file containing a list of dicts containing the username and - emails of the contributors in the given github project - ''' - - github_obj = Github(github_username, github_password) - project = github_obj.get_repo(github_project_name) - commits_url = project.commits_url.replace('{/sha}', '') - - page = 0 - contributors = [] - while True: - page += 1 - payload = {'page': page } - data_ = json.loads(requests.get(commits_url, params=payload, - auth=HTTPBasicAuth(github_username, github_password)).text) - - if not data_: - break - - for data in data_: - try: - contributor = data['commit']['committer'] - contributor_email = contributor['email'] - contributor_fullname = contributor['name'] - contributor_name = data['committer']['login'] - except TypeError: - print 'Maybe one of the contributors is dropped because of lack of details' - continue - - json_data = { - 'name': contributor_name, - 'fullname': contributor_fullname, - 'emails': [contributor_email] - } - - present = False - for i in contributors: - if i == json_data: - present = True - break - - if not present: - print 'contributor added: ', contributor_name - contributors.append(json_data) - - with open('contributors.json', 'w') as f: - f.write(json.dumps(contributors)) - - return - - -def generate_json_for_github_issue_commentors(github_username, github_password, \ - github_project_name): - ''' Will create a json file containing details of all the user - who have commented on any issue in the given project - ''' - - github_obj = Github(github_username, github_password) - project = github_obj.get_repo(github_project_name) - issue_comment_url = project.issue_comment_url.replace('{/number}', '') - - page = 0 - issue_commentors = [] - while True: - page += 1 - payload = {'page': page } - data_ = json.loads(requests.get(issue_comment_url, params=payload, - auth=HTTPBasicAuth(github_username, github_password)).text) - - if not data_: - break - - for data in data_: - try: - commentor = data['user']['login'] - except TypeError: - print 'Maybe one of the issue commentors have been dropped because of lack of details' - continue - - present = False - for i in issue_commentors: - if i == commentor: - present = True - break - - if not present: - print 'commentor added: ', commentor - issue_commentors.append(commentor) - - with open('issue_commentors.json', 'w') as f: - f.write(json.dumps(issue_commentors)) - return - - -def assemble_github_contributors_commentors(): - ''' It uses the files: issue_commentors.json and contributors.json - Assembles and creates a file: assembled_commentors.csv - To use: just fill the empty blocks under emails column''' - - with open('issue_commentors.json', 'r') as ic: - issue_names = json.load(ic) - - with open('contributors.json', 'r') as c: - contributors = json.load(c) - - names = [] - for i in issue_names: - found = False - for j in contributors: - if j.get('name', None) == i: - j['emails'] = j.get('emails')[0] - names.append(j) - found = True - - if not found: - d = {'name': i, 'fullname': None, 'emails': None} - names.append(d) - - with open('assembled_commentors.csv', 'w') as ac: - field_names = ['name', 'fullname', 'emails'] - writer = csv.DictWriter(ac, fieldnames=field_names) - - writer.writeheader() - for name in names: - writer.writerow(name) - - -def github_get_commentor_email(name): - ''' Will return the issue commentor email as given in the - assembled_commentors.csv file - ''' - - if not os.path.exists('assembled_commentors.csv'): - raise FileNotFound('The assembled_commentors.json file must be present \ - Rerun the program and choose to generate the json files') - - data = [] - with open('assembled_commentors.csv') as ac: - reader = csv.DictReader(ac) - for row in reader: - data.append(dict( \ - (('name', row['name']), \ - ('fullname', row['fullname']), \ - ('emails', row['emails'])))) - - - for i in data: - if i.get('name', None) == name: - if i['emails']: - return str(i['emails']) - else: - raise EmailNotFound('You need to fill out all the emails of the \ - issue commentors') - diff --git a/pagure_importer/lib/exceptions.py b/pagure_importer/lib/exceptions.py deleted file mode 100644 index a5ab5fe..0000000 --- a/pagure_importer/lib/exceptions.py +++ /dev/null @@ -1,22 +0,0 @@ -class GithubBadCredentials(Exception): - ''' Raised when username/password for github is wrong ''' - def __init__(self, msg): - self.msg = msg - - -class GithubRepoNotFound(Exception): - ''' Raised when the repo is not found for the user ''' - def __init__(self, msg): - self.msg = msg - - -class FileNotFound(Exception): - ''' Raised when a certain file is not found ''' - def __init__(self, msg): - self.msg = msg - - -class EmailNotFound(Exception): - ''' Raised when email is not found ''' - def __init__(self, msg): - self.msg = msg diff --git a/pagure_importer/lib/fas.py b/pagure_importer/lib/fas.py deleted file mode 100644 index b63f665..0000000 --- a/pagure_importer/lib/fas.py +++ /dev/null @@ -1,23 +0,0 @@ -from fedora.client.fas2 import AccountSystem -from pagure_importer.lib.models import User - - -class FASclient (): - def __init__(self, fas_username, fas_password, fas_url): - self.fasclient = AccountSystem(fas_url, username=fas_username, - password=fas_password) - - anonymous = User(name='', fullname='', emails=[]) - self.fasuser = {'': anonymous} - - def find_fas_user(self, user): - - if user not in self.fasuser.keys(): - person = self.fasclient.person_by_username(user) - if not person: - return self.fasuser[''] - - self.fasuser[user] = User(name=user, - fullname=person['human_name'], - emails=[person['email']]) - return self.fasuser[user] diff --git a/pagure_importer/lib/git.py b/pagure_importer/lib/git.py deleted file mode 100644 index 33eba48..0000000 --- a/pagure_importer/lib/git.py +++ /dev/null @@ -1,98 +0,0 @@ -''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py - by pingou@pingoured.fr -''' - -import shutil -import os -import pygit2 -import tempfile -import json - -from repo import * - -def update_git(obj, repo_path, repo_folder): - """ Update the given issue in its git. - This method forks the provided repo, add/edit the issue whose file name - is defined by the uid field of the issue and if there are additions/ - changes commit them and push them back to the original repo. - """ - - if not repo_folder: - return - - # Get the fork - repopath = os.path.join(repo_folder, repo_path) - - # Clone the repo into a temp folder - newpath = tempfile.mkdtemp(prefix='pagure-') - new_repo = pygit2.clone_repository(repopath, newpath) - - file_path = os.path.join(newpath, obj.uid) - - # Get the current index - index = new_repo.index - - # Are we adding files - added = False - if not os.path.exists(file_path): - added = True - - # Write down what changed - with open(file_path, 'w') as stream: - stream.write(json.dumps( - obj.to_json(), sort_keys=True, indent=4, - separators=(',', ': '))) - - # Retrieve the list of files that changed - diff = new_repo.diff() - files = [] - for p in diff: - if hasattr(p, 'new_file_path'): - files.append(p.new_file_path) - elif hasattr(p, 'delta'): - files.append(p.delta.new_file.path) - - # Add the changes to the index - if added: - index.add(obj.uid) - for filename in files: - index.add(filename) - - # If not change, return - if not files and not added: - shutil.rmtree(newpath) - return - - # See if there is a parent to this commit - parent = None - try: - parent = new_repo.head.get_object().oid - except pygit2.GitError: - pass - - parents = [] - if parent: - parents.append(parent) - - # Author/commiter will always be this one - author = pygit2.Signature(name='pagure', email='pagure') - - # Actually commit - new_repo.create_commit( - 'refs/heads/master', - author, - author, - 'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title), - new_repo.index.write_tree(), - parents) - index.write() - - # Push to origin - ori_remote = new_repo.remotes[0] - master_ref = new_repo.lookup_reference('HEAD').resolve() - refname = '%s:%s' % (master_ref.name, master_ref.name) - - PagureRepo.push(ori_remote, refname) - - # Remove the clone - shutil.rmtree(newpath) diff --git a/pagure_importer/lib/models.py b/pagure_importer/lib/models.py deleted file mode 100644 index 3aa3d83..0000000 --- a/pagure_importer/lib/models.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding: utf-8 -*- - -import datetime -import json -import uuid - -class Issue(): - ''' Represents an Issue ''' - - def __init__( - self, id, title, content, - status, date_created, user, private, tags, - depends, blocks, assignee, comments=None): - - self.id = id - self.title = title - self.content = content - self.status = status - self.date_created = date_created - self.user = user - self.private = private - self.tags = tags - self.depends = depends - self.blocks = blocks - self.assignee = assignee - self.comments = comments - self.uid = uuid.uuid4().hex - - def to_json(self): - ''' Returns a dictionary representation of the issue. - - ''' - output = { - 'id': self.id, - 'title': self.title, - 'content': self.content, - 'status': self.status, - 'date_created': self.date_created.strftime('%s'), - 'user': self.user, - 'private': self.private, - 'tags': self.tags, - 'depends': self.depends, - 'blocks': self.blocks, - 'assignee': self.assignee, - 'comments': self.comments - } - - return output - - @property - def isa(self): - return 'issue' - - -class IssueComment(): - ''' Represent a comment for an issue ''' - - def __init__( - self, id, comment, date_created, - user, parent=None, edited_on=None, editor=None): - - self.id = id - self.comment = comment - self.parent = parent - self.date_created = date_created - self.user = user - self.edited_on = edited_on - self.editor = editor - - def to_json(self): - ''' Returns a dictionary representation of the issue. ''' - - output = { - 'id': self.id, - 'comment': self.comment, - 'parent': self.parent, - 'date_created': self.date_created.strftime('%s'), - 'user': self.user, - 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, - 'editor': self.editor or None - } - - return output - - -class User(): - ''' Represents a User ''' - - def __init__( - self, name, emails, - fullname=None): - self.name = name - self.fullname = fullname - self.emails = emails - - def to_json(self): - ''' Return a representation of the User in a dictionary. ''' - - output = { - 'name': self.name, - 'fullname': self.fullname, - 'emails': self.emails - } - - return output diff --git a/pagure_importer/lib/repo.py b/pagure_importer/lib/repo.py deleted file mode 100644 index f46b31a..0000000 --- a/pagure_importer/lib/repo.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- - -''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py - by pingou@pingoured.fr -''' - - -import pygit2 -import sys - - -def get_pygit2_version(): - ''' Return pygit2 version as a tuple of integers. - This is needed for correct version comparison. - ''' - return tuple([int(i) for i in pygit2.__version__.split('.')]) - - -class PagureRepo(pygit2.Repository): - """ An utility class allowing to go around pygit2's inability to be - stable. - - """ - - @staticmethod - def push(remote, refname): - """ Push the given reference to the specified remote. """ - pygit2_version = get_pygit2_version() - if pygit2_version >= (0, 22): - remote.push([refname]) - else: - remote.push(refname) - - def pull(self, remote_name='origin', branch='master', force=False): - ''' pull changes for the specified remote (defaults to origin). - - Code from MichaelBoselowitz at: - https://github.com/MichaelBoselowitz/pygit2-examples/blob/ - 68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58 - licensed under the MIT license. - ''' - - for remote in self.remotes: - if remote.name == remote_name: - remote.fetch() - remote_master_id = self.lookup_reference( - 'refs/remotes/origin/%s' % branch).target - - if force: - repo_branch = self.lookup_reference( - 'refs/heads/%s' % branch) - repo_branch.set_target(remote_master_id) - - merge_result, _ = self.merge_analysis(remote_master_id) - # Up to date, do nothing - if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: - return - # We can just fastforward - elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: - self.checkout_tree(self.get(remote_master_id)) - master_ref = self.lookup_reference( - 'refs/heads/%s' % branch) - master_ref.set_target(remote_master_id) - self.head.set_target(remote_master_id) - elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: - sys.exit('Pulling remote changes leads to a conflict') - else: - print 'Unexpected merge result: %s' % ( - pygit2.GIT_MERGE_ANALYSIS_NORMAL) - raise AssertionError('Unknown merge analysis result') diff --git a/pagure_importer/lib/sources/__init__.py b/pagure_importer/lib/sources/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/pagure_importer/lib/sources/__init__.py +++ /dev/null diff --git a/pagure_importer/lib/sources/importer_github.py b/pagure_importer/lib/sources/importer_github.py deleted file mode 100644 index 2859e99..0000000 --- a/pagure_importer/lib/sources/importer_github.py +++ /dev/null @@ -1,134 +0,0 @@ -from github import Github -import pagure_importer -import pagure_importer.lib -from pagure_importer.lib import models -from pagure_importer.lib import github_get_commentor_email -from pagure_importer.lib.exceptions import GithubBadCredentials, GithubRepoNotFound - -class GithubImporter(): - ''' Imports from Github using PyGithub and libpagure ''' - def __init__( - self, - github_username, - github_password, - github_project_name): - self.github_username = github_username - self.github_password = github_password - self.github_project_name = github_project_name - self.github = Github(github_username, github_password) - - def import_issues(self, repo_path, repo_folder, status='all'): - ''' Imports the issues on github for - the given project - ''' - github_user = None - try: - github_user = self.github.get_user(self.github_username) - except: - raise GithubBadCredentials( - 'Given github credentials are not correct') - repo = self.github.get_repo(self.github_project_name) - try: - repo_name = repo.name - except: - raise GithubRepoNotFound( - 'Repo not found, project name wrong') - - for github_issue in repo.get_issues(state=status): - - #title of the issue - pagure_issue_title = github_issue.title - - #body of the issue - if github_issue.body: - pagure_issue_content = github_issue.body - else: - pagure_issue_content = '#No Description Provided' - - #Some details of a issue - if github_issue.state != 'closed': - pagure_issue_status = 'Open' - else: - pagure_issue_status = 'Fixed' - - pagure_issue_created_at = github_issue.created_at - - #Not sure how to deal with this atm - pagure_issue_assignee = None - - if github_issue.labels: - pagure_issue_tags = [i.name for i in github_issue.labels] - else: - pagure_issue_tags = [] - - - #few things not supported by github - pagure_issue_depends = [] - pagure_issue_blocks = [] - pagure_issue_is_private = False - - - #User who created the issue - pagure_issue_user = models.User( - name=github_issue.user.login, - fullname=github_issue.user.name, - emails=[github_issue.user.email]) - - - pagure_issue = models.Issue( - id=None, - title = pagure_issue_title, - content = pagure_issue_content, - status = pagure_issue_status, - date_created = pagure_issue_created_at, - user = pagure_issue_user.to_json(), - private = pagure_issue_is_private, - tags = pagure_issue_tags, - depends = pagure_issue_depends, - blocks = pagure_issue_blocks, - assignee = pagure_issue_assignee) - - - #comments on the issue - comments = [] - for comment in github_issue.get_comments(): - - comment_user = comment.user - pagure_issue_comment_user_email = comment_user.email - pagure_issue_comment_body = comment.body - pagure_issue_comment_created_at = comment.created_at - pagure_issue_comment_updated_at = comment.updated_at - - - #No idea what to do with this right now - #editor: not supported by github api - pagure_issue_comment_parent = None - pagure_issue_comment_editor = None - - #comment updated at - pagure_issue_comment_edited_on = comment.updated_at - - #The User who commented - pagure_issue_comment_user = models.User( - name=comment_user.login, - fullname=comment_user.name, - emails=[comment_user.email] if comment_user.email \ - else [github_get_commentor_email(comment_user.login)]) - - #Object to represent comment on an issue - pagure_issue_comment = models.IssueComment( - id=None, - comment=pagure_issue_comment_body, - parent=pagure_issue_comment_parent, - date_created=pagure_issue_comment_created_at, - user=pagure_issue_comment_user.to_json(), - edited_on=pagure_issue_comment_edited_on, - editor=pagure_issue_comment_editor) - - comments.append(pagure_issue_comment.to_json()) - - #add all the comments to the issue object - pagure_issue.comments = comments - - #update the local git repo - pagure_importer.lib.git.update_git(pagure_issue, repo_path, repo_folder) diff --git a/pagure_importer/lib/sources/importer_trac.py b/pagure_importer/lib/sources/importer_trac.py deleted file mode 100644 index f69a3ce..0000000 --- a/pagure_importer/lib/sources/importer_trac.py +++ /dev/null @@ -1,132 +0,0 @@ -from xmlrpclib import ServerProxy -import pagure_importer -import pagure_importer.lib -from pagure_importer.lib.fas import FASclient -from pagure_importer.lib import trac - - -class TracImporter(): - '''Pagure importer for trac instance''' - - def __init__(self, trac_project_url): - self.tracclient = ServerProxy(trac_project_url + '/rpc') - self.fasclient = FASclient('user', 'password', - 'https://admin.fedoraproject.org/accounts') - - def _find_fas_user(self, user): - person = self.fasclient.person_by_username(user) - human_name = person['human_name'] - email = person['email'] - pagure_user = models.User( - name=user, - fullname=human_name, - emails=[email]) - return pagure_user - - def _get_ticket_tags(self, trac_ticket): - return [] - - def _get_ticket_status(self, trac_ticket): - ''' Converts Trac ticket status - to Pagure issue status''' - - if trac_ticket['status'] != 'closed': - ticket_status = 'Open' - else: - ticket_status = 'Fixed' - return ticket_status - - def _populate_comments(self, trac_comments): - comments = [] - for comment in trac_comments: - if comment[2] == 'comment' and comment[4] != '': - comment_user = comment[1] - pagure_issue_comment_user_email = None - pagure_issue_comment_body = comment[4] - pagure_issue_comment_created_at = datetime.strptime( - comment[0].value, "%Y%m%dT%H:%M:%S") - pagure_issue_comment_updated_at = None - - # No idea what to do with this right now - # editor: not supported by github api - pagure_issue_comment_parent = None - pagure_issue_comment_editor = None - - # comment updated at - pagure_issue_comment_edited_on = None - - # The User who commented - pagure_issue_comment_user = self._find_fas_user(comment[1]) - - # Object to represent comment on an issue - pagure_issue_comment = models.IssueComment( - id=None, - comment=pagure_issue_comment_body, - parent=pagure_issue_comment_parent, - date_created=pagure_issue_comment_created_at, - user=pagure_issue_comment_user.to_json(), - edited_on=pagure_issue_comment_edited_on, - editor=pagure_issue_comment_editor) - - comments.append(pagure_issue_comment.to_json()) - return comments - - def _populate_issue(self, ticket_id): - trac_ticket = self.trac.ticket.get(ticket_id)[3] - pagure_issue_title = trac_ticket['summary'] - pagure_issue_content = trac_ticket['description'] - - if pagure_issue_content == '': - pagure_issue_content = '#No Description Provided' - - pagure_issue_status = self._get_ticket_status(trac_ticket) - - pagure_issue_created_at = datetime.strptime( - self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") - - pagure_issue_assignee = trac_ticket['owner'] - - pagure_issue_tags = self._get_ticket_tags(trac_ticket) - - pagure_issue_depends = [] - pagure_issue_blocks = [] - pagure_issue_is_private = False - - pagure_issue_user = self._find_fas_user(trac_ticket['reporter']) - - pagure_issue = models.Issue( - id=None, - title=pagure_issue_title, - content=pagure_issue_content, - status=pagure_issue_status, - date_created=pagure_issue_created_at, - user=pagure_issue_user.to_json(), - private=pagure_issue_is_private, - tags=pagure_issue_tags, - depends=pagure_issue_depends, - blocks=pagure_issue_blocks, - assignee=pagure_issue_assignee) - return pagure_issue - - def import_issues(self, repo_path, repo_folder, - trac_query='max=0&order=id'): - '''Import issues from trac instance using xmlrpc API''' - tickets_id = self.tracclient.ticket.query(trac_query) - - for ticket_id in tickets_id: - - pagure_issue = trac.populate_issue(self.tracclient, - self.fasclient, ticket_id) - - pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) - comments = trac.populate_comments(self.fasclient, - pagure_issue_comments) - - # add all the comments to the issue object - pagure_issue.comments = comments - - # update the local git repo - print 'Update repo with issue :' + str(ticket_id) - pagure_importer.lib.git.update_git(pagure_issue, - repo_path, - repo_folder) diff --git a/pagure_importer/lib/trac.py b/pagure_importer/lib/trac.py deleted file mode 100644 index fb3d062..0000000 --- a/pagure_importer/lib/trac.py +++ /dev/null @@ -1,90 +0,0 @@ -from pagure_importer.lib.models import IssueComment, Issue -from datetime import datetime - - -def get_ticket_tags(trac_ticket): - return [] - - -def get_ticket_status(trac_ticket): - ''' Converts Trac ticket status - to Pagure issue status''' - - if trac_ticket['status'] != 'closed': - ticket_status = 'Open' - else: - ticket_status = 'Fixed' - return ticket_status - - -def populate_comments(fasclient, trac_comments): - comments = [] - for comment in trac_comments: - if comment[2] == 'comment' and comment[4] != '': - comment_user = comment[1] - pagure_issue_comment_user_email = None - pagure_issue_comment_body = comment[4] - pagure_issue_comment_created_at = datetime.strptime( - comment[0].value, "%Y%m%dT%H:%M:%S") - pagure_issue_comment_updated_at = None - - # No idea what to do with this right now - # editor: not supported by github api - pagure_issue_comment_parent = None - pagure_issue_comment_editor = None - - # comment updated at - pagure_issue_comment_edited_on = None - - # The User who commented - pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) - - # Object to represent comment on an issue - pagure_issue_comment = IssueComment( - id=None, - comment=pagure_issue_comment_body, - parent=pagure_issue_comment_parent, - date_created=pagure_issue_comment_created_at, - user=pagure_issue_comment_user.to_json(), - edited_on=pagure_issue_comment_edited_on, - editor=pagure_issue_comment_editor) - - comments.append(pagure_issue_comment.to_json()) - return comments - - -def populate_issue(trac, fasclient, ticket_id): - trac_ticket = trac.ticket.get(ticket_id)[3] - pagure_issue_title = trac_ticket['summary'] - pagure_issue_content = trac_ticket['description'] - - if pagure_issue_content == '': - pagure_issue_content = '#No Description Provided' - - pagure_issue_status = get_ticket_status(trac_ticket) - - pagure_issue_created_at = datetime.strptime( - trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") - - pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) - - pagure_issue_tags = get_ticket_tags(trac_ticket) - - pagure_issue_depends = [] - pagure_issue_blocks = [] - pagure_issue_is_private = False - - pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) - pagure_issue = Issue( - id=ticket_id, - title=pagure_issue_title, - content=pagure_issue_content, - status=pagure_issue_status, - date_created=pagure_issue_created_at, - user=pagure_issue_user.to_json(), - private=pagure_issue_is_private, - tags=pagure_issue_tags, - depends=pagure_issue_depends, - blocks=pagure_issue_blocks, - assignee=pagure_issue_assignee.to_json()) - return pagure_issue diff --git a/pagure_importer/run.py b/pagure_importer/run.py deleted file mode 100644 index d327228..0000000 --- a/pagure_importer/run.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -import getpass -from forms import form_github_issues -from settings import IMPORT_SOURCES, IMPORT_OPTIONS, REPO_NAME, REPO_PATH -import pagure_importer -import pagure_importer.lib -import pagure_importer.lib.sources -from pagure_importer.lib.sources.importer_github import GithubImporter -from pagure_importer.lib.sources.importer_trac import TracImporter -from pagure_importer.lib import generate_json_for_github_contributors, \ - generate_json_for_github_issue_commentors, \ - assemble_github_contributors_commentors - - -def github_handler(item): - if item.lower() == 'issues': - github_username, github_password, github_project_name = form_github_issues() - gen_json = raw_input('Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') - if gen_json == 'n': - github_importer = GithubImporter( - github_username=github_username, - github_password=github_password, - github_project_name=github_project_name) - github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) - else: - generate_json_for_github_contributors( - github_username, - github_password, - github_project_name) - generate_json_for_github_issue_commentors( - github_username, - github_password, - github_project_name) - assemble_github_contributors_commentors() - return - - -def trac_handler(item, fedora=False): - if item.lower() == 'issues': - trac_url = raw_input('Enter the trac project url: ') - trac_importer = TracImporter(trac_url) - trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) - - -def main(): - source = raw_input('Enter source from where you want to import: ') - if source.lower() not in IMPORT_SOURCES: - print 'Source location not supported' - return - - item = raw_input('Enter the item to be imported: ') - if item.lower() not in IMPORT_OPTIONS[source]: - print 'Item import not supported' - return - - if source.lower() == 'github': - github_handler(item) - elif source.lower() == 'fedorahosted': - trac_handler(item, fedora=True) - return - -if __name__ == '__main__': - main() diff --git a/pagure_importer/settings.py b/pagure_importer/settings.py deleted file mode 100644 index 077ea40..0000000 --- a/pagure_importer/settings.py +++ /dev/null @@ -1,7 +0,0 @@ -import os - -IMPORT_SOURCES = ['github', 'fedorahosted'] -IMPORT_OPTIONS = {'github': ['issues'], 'fedorahosted': ['issues']} - -REPO_NAME = os.environ.get('REPO_NAME', None) #this has to be a bare repo -REPO_PATH = os.environ.get('REPO_PATH', None) #the parent of the git directory diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py new file mode 100644 index 0000000..05ccdb8 --- /dev/null +++ b/pagure_importer/utils/__init__.py @@ -0,0 +1,168 @@ +import git +import models + +import csv +import os +import getpass +import requests +import json +from github import Github +from requests.auth import HTTPBasicAuth + +from pagure_importer.utils.exceptions import FileNotFound, EmailNotFound + +def generate_json_for_github_contributors(github_username, github_password, \ + github_project_name): + ''' Creates a file containing a list of dicts containing the username and + emails of the contributors in the given github project + ''' + + github_obj = Github(github_username, github_password) + project = github_obj.get_repo(github_project_name) + commits_url = project.commits_url.replace('{/sha}', '') + + page = 0 + contributors = [] + while True: + page += 1 + payload = {'page': page } + data_ = json.loads(requests.get(commits_url, params=payload, + auth=HTTPBasicAuth(github_username, github_password)).text) + + if not data_: + break + + for data in data_: + try: + contributor = data['commit']['committer'] + contributor_email = contributor['email'] + contributor_fullname = contributor['name'] + contributor_name = data['committer']['login'] + except TypeError: + print 'Maybe one of the contributors is dropped because of lack of details' + continue + + json_data = { + 'name': contributor_name, + 'fullname': contributor_fullname, + 'emails': [contributor_email] + } + + present = False + for i in contributors: + if i == json_data: + present = True + break + + if not present: + print 'contributor added: ', contributor_name + contributors.append(json_data) + + with open('contributors.json', 'w') as f: + f.write(json.dumps(contributors)) + + return + + +def generate_json_for_github_issue_commentors(github_username, github_password, \ + github_project_name): + ''' Will create a json file containing details of all the user + who have commented on any issue in the given project + ''' + + github_obj = Github(github_username, github_password) + project = github_obj.get_repo(github_project_name) + issue_comment_url = project.issue_comment_url.replace('{/number}', '') + + page = 0 + issue_commentors = [] + while True: + page += 1 + payload = {'page': page } + data_ = json.loads(requests.get(issue_comment_url, params=payload, + auth=HTTPBasicAuth(github_username, github_password)).text) + + if not data_: + break + + for data in data_: + try: + commentor = data['user']['login'] + except TypeError: + print 'Maybe one of the issue commentors have been dropped because of lack of details' + continue + + present = False + for i in issue_commentors: + if i == commentor: + present = True + break + + if not present: + print 'commentor added: ', commentor + issue_commentors.append(commentor) + + with open('issue_commentors.json', 'w') as f: + f.write(json.dumps(issue_commentors)) + return + + +def assemble_github_contributors_commentors(): + ''' It uses the files: issue_commentors.json and contributors.json + Assembles and creates a file: assembled_commentors.csv + To use: just fill the empty blocks under emails column''' + + with open('issue_commentors.json', 'r') as ic: + issue_names = json.load(ic) + + with open('contributors.json', 'r') as c: + contributors = json.load(c) + + names = [] + for i in issue_names: + found = False + for j in contributors: + if j.get('name', None) == i: + j['emails'] = j.get('emails')[0] + names.append(j) + found = True + + if not found: + d = {'name': i, 'fullname': None, 'emails': None} + names.append(d) + + with open('assembled_commentors.csv', 'w') as ac: + field_names = ['name', 'fullname', 'emails'] + writer = csv.DictWriter(ac, fieldnames=field_names) + + writer.writeheader() + for name in names: + writer.writerow(name) + + +def github_get_commentor_email(name): + ''' Will return the issue commentor email as given in the + assembled_commentors.csv file + ''' + + if not os.path.exists('assembled_commentors.csv'): + raise FileNotFound('The assembled_commentors.json file must be present \ + Rerun the program and choose to generate the json files') + + data = [] + with open('assembled_commentors.csv') as ac: + reader = csv.DictReader(ac) + for row in reader: + data.append(dict( \ + (('name', row['name']), \ + ('fullname', row['fullname']), \ + ('emails', row['emails'])))) + + + for i in data: + if i.get('name', None) == name: + if i['emails']: + return str(i['emails']) + else: + raise EmailNotFound('You need to fill out all the emails of the \ + issue commentors') diff --git a/pagure_importer/utils/exceptions.py b/pagure_importer/utils/exceptions.py new file mode 100644 index 0000000..a5ab5fe --- /dev/null +++ b/pagure_importer/utils/exceptions.py @@ -0,0 +1,22 @@ +class GithubBadCredentials(Exception): + ''' Raised when username/password for github is wrong ''' + def __init__(self, msg): + self.msg = msg + + +class GithubRepoNotFound(Exception): + ''' Raised when the repo is not found for the user ''' + def __init__(self, msg): + self.msg = msg + + +class FileNotFound(Exception): + ''' Raised when a certain file is not found ''' + def __init__(self, msg): + self.msg = msg + + +class EmailNotFound(Exception): + ''' Raised when email is not found ''' + def __init__(self, msg): + self.msg = msg diff --git a/pagure_importer/utils/fas.py b/pagure_importer/utils/fas.py new file mode 100644 index 0000000..f62b5f5 --- /dev/null +++ b/pagure_importer/utils/fas.py @@ -0,0 +1,23 @@ +from fedora.client.fas2 import AccountSystem +from pagure_importer.utils.models import User + + +class FASclient (): + def __init__(self, fas_username, fas_password, fas_url): + self.fasclient = AccountSystem(fas_url, username=fas_username, + password=fas_password) + + anonymous = User(name='', fullname='', emails=[]) + self.fasuser = {'': anonymous} + + def find_fas_user(self, user): + + if user not in self.fasuser.keys(): + person = self.fasclient.person_by_username(user) + if not person: + return self.fasuser[''] + + self.fasuser[user] = User(name=user, + fullname=person['human_name'], + emails=[person['email']]) + return self.fasuser[user] diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py new file mode 100644 index 0000000..33eba48 --- /dev/null +++ b/pagure_importer/utils/git.py @@ -0,0 +1,98 @@ +''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py + by pingou@pingoured.fr +''' + +import shutil +import os +import pygit2 +import tempfile +import json + +from repo import * + +def update_git(obj, repo_path, repo_folder): + """ Update the given issue in its git. + This method forks the provided repo, add/edit the issue whose file name + is defined by the uid field of the issue and if there are additions/ + changes commit them and push them back to the original repo. + """ + + if not repo_folder: + return + + # Get the fork + repopath = os.path.join(repo_folder, repo_path) + + # Clone the repo into a temp folder + newpath = tempfile.mkdtemp(prefix='pagure-') + new_repo = pygit2.clone_repository(repopath, newpath) + + file_path = os.path.join(newpath, obj.uid) + + # Get the current index + index = new_repo.index + + # Are we adding files + added = False + if not os.path.exists(file_path): + added = True + + # Write down what changed + with open(file_path, 'w') as stream: + stream.write(json.dumps( + obj.to_json(), sort_keys=True, indent=4, + separators=(',', ': '))) + + # Retrieve the list of files that changed + diff = new_repo.diff() + files = [] + for p in diff: + if hasattr(p, 'new_file_path'): + files.append(p.new_file_path) + elif hasattr(p, 'delta'): + files.append(p.delta.new_file.path) + + # Add the changes to the index + if added: + index.add(obj.uid) + for filename in files: + index.add(filename) + + # If not change, return + if not files and not added: + shutil.rmtree(newpath) + return + + # See if there is a parent to this commit + parent = None + try: + parent = new_repo.head.get_object().oid + except pygit2.GitError: + pass + + parents = [] + if parent: + parents.append(parent) + + # Author/commiter will always be this one + author = pygit2.Signature(name='pagure', email='pagure') + + # Actually commit + new_repo.create_commit( + 'refs/heads/master', + author, + author, + 'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title), + new_repo.index.write_tree(), + parents) + index.write() + + # Push to origin + ori_remote = new_repo.remotes[0] + master_ref = new_repo.lookup_reference('HEAD').resolve() + refname = '%s:%s' % (master_ref.name, master_ref.name) + + PagureRepo.push(ori_remote, refname) + + # Remove the clone + shutil.rmtree(newpath) diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py new file mode 100644 index 0000000..cc61370 --- /dev/null +++ b/pagure_importer/utils/importer_github.py @@ -0,0 +1,138 @@ +from github import Github + +from pagure_importer.utils import models +from pagure_importer.utils import github_get_commentor_email +from pagure_importer.utils.git import update_git +from pagure_importer.utils.exceptions import ( + GithubBadCredentials, + GithubRepoNotFound +) + + +class GithubImporter(): + ''' Imports from Github using PyGithub and libpagure ''' + def __init__( + self, + github_username, + github_password, + github_project_name): + self.github_username = github_username + self.github_password = github_password + self.github_project_name = github_project_name + self.github = Github(github_username, github_password) + + def import_issues(self, repo_path, repo_folder, status='all'): + ''' Imports the issues on github for + the given project + ''' + github_user = None + try: + github_user = self.github.get_user(self.github_username) + except: + raise GithubBadCredentials( + 'Given github credentials are not correct') + repo = self.github.get_repo(self.github_project_name) + try: + repo_name = repo.name + except: + raise GithubRepoNotFound( + 'Repo not found, project name wrong') + + for github_issue in repo.get_issues(state=status): + + # title of the issue + pagure_issue_title = github_issue.title + + # body of the issue + if github_issue.body: + pagure_issue_content = github_issue.body + else: + pagure_issue_content = '#No Description Provided' + + # Some details of a issue + if github_issue.state != 'closed': + pagure_issue_status = 'Open' + else: + pagure_issue_status = 'Fixed' + + pagure_issue_created_at = github_issue.created_at + + # Not sure how to deal with this atm + pagure_issue_assignee = None + + if github_issue.labels: + pagure_issue_tags = [i.name for i in github_issue.labels] + else: + pagure_issue_tags = [] + + + # few things not supported by github + pagure_issue_depends = [] + pagure_issue_blocks = [] + pagure_issue_is_private = False + + + # User who created the issue + pagure_issue_user = models.User( + name=github_issue.user.login, + fullname=github_issue.user.name, + emails=[github_issue.user.email]) + + + pagure_issue = models.Issue( + id=None, + title = pagure_issue_title, + content = pagure_issue_content, + status = pagure_issue_status, + date_created = pagure_issue_created_at, + user = pagure_issue_user.to_json(), + private = pagure_issue_is_private, + tags = pagure_issue_tags, + depends = pagure_issue_depends, + blocks = pagure_issue_blocks, + assignee = pagure_issue_assignee) + + + # comments on the issue + comments = [] + for comment in github_issue.get_comments(): + + comment_user = comment.user + pagure_issue_comment_user_email = comment_user.email + pagure_issue_comment_body = comment.body + pagure_issue_comment_created_at = comment.created_at + pagure_issue_comment_updated_at = comment.updated_at + + + # No idea what to do with this right now + # editor: not supported by github api + pagure_issue_comment_parent = None + pagure_issue_comment_editor = None + + # comment updated at + pagure_issue_comment_edited_on = comment.updated_at + + # The User who commented + pagure_issue_comment_user = models.User( + name=comment_user.login, + fullname=comment_user.name, + emails=[comment_user.email] if comment_user.email \ + else [github_get_commentor_email(comment_user.login)]) + + # Object to represent comment on an issue + pagure_issue_comment = models.IssueComment( + id=None, + comment=pagure_issue_comment_body, + parent=pagure_issue_comment_parent, + date_created=pagure_issue_comment_created_at, + user=pagure_issue_comment_user.to_json(), + edited_on=pagure_issue_comment_edited_on, + editor=pagure_issue_comment_editor) + + comments.append(pagure_issue_comment.to_json()) + + # add all the comments to the issue object + pagure_issue.comments = comments + + # update the local git repo + update_git(pagure_issue, repo_path, repo_folder) diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py new file mode 100644 index 0000000..8e7fff8 --- /dev/null +++ b/pagure_importer/utils/importer_trac.py @@ -0,0 +1,32 @@ +from xmlrpclib import ServerProxy +from pagure_importer.utils.git import update_git +from pagure_importer.utils import trac + + +class TracImporter(): + '''Pagure importer for trac instance''' + + def __init__(self, trac_project_url, fasclient=None): + self.tracclient = ServerProxy(trac_project_url + '/rpc') + self.fasclient = fasclient + + def import_issues(self, repo_path, repo_folder, + trac_query='max=0&order=id'): + '''Import issues from trac instance using xmlrpc API''' + tickets_id = self.tracclient.ticket.query(trac_query) + + for ticket_id in tickets_id: + + pagure_issue = trac.populate_issue(self.tracclient, + self.fasclient, ticket_id) + + pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) + comments = trac.populate_comments(self.fasclient, + pagure_issue_comments) + + # add all the comments to the issue object + pagure_issue.comments = comments + + # update the local git repo + print 'Update repo with issue :' + str(ticket_id) + update_git(pagure_issue, repo_path, repo_folder) diff --git a/pagure_importer/utils/models.py b/pagure_importer/utils/models.py new file mode 100644 index 0000000..3aa3d83 --- /dev/null +++ b/pagure_importer/utils/models.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- + +import datetime +import json +import uuid + +class Issue(): + ''' Represents an Issue ''' + + def __init__( + self, id, title, content, + status, date_created, user, private, tags, + depends, blocks, assignee, comments=None): + + self.id = id + self.title = title + self.content = content + self.status = status + self.date_created = date_created + self.user = user + self.private = private + self.tags = tags + self.depends = depends + self.blocks = blocks + self.assignee = assignee + self.comments = comments + self.uid = uuid.uuid4().hex + + def to_json(self): + ''' Returns a dictionary representation of the issue. + + ''' + output = { + 'id': self.id, + 'title': self.title, + 'content': self.content, + 'status': self.status, + 'date_created': self.date_created.strftime('%s'), + 'user': self.user, + 'private': self.private, + 'tags': self.tags, + 'depends': self.depends, + 'blocks': self.blocks, + 'assignee': self.assignee, + 'comments': self.comments + } + + return output + + @property + def isa(self): + return 'issue' + + +class IssueComment(): + ''' Represent a comment for an issue ''' + + def __init__( + self, id, comment, date_created, + user, parent=None, edited_on=None, editor=None): + + self.id = id + self.comment = comment + self.parent = parent + self.date_created = date_created + self.user = user + self.edited_on = edited_on + self.editor = editor + + def to_json(self): + ''' Returns a dictionary representation of the issue. ''' + + output = { + 'id': self.id, + 'comment': self.comment, + 'parent': self.parent, + 'date_created': self.date_created.strftime('%s'), + 'user': self.user, + 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, + 'editor': self.editor or None + } + + return output + + +class User(): + ''' Represents a User ''' + + def __init__( + self, name, emails, + fullname=None): + self.name = name + self.fullname = fullname + self.emails = emails + + def to_json(self): + ''' Return a representation of the User in a dictionary. ''' + + output = { + 'name': self.name, + 'fullname': self.fullname, + 'emails': self.emails + } + + return output diff --git a/pagure_importer/utils/repo.py b/pagure_importer/utils/repo.py new file mode 100644 index 0000000..f46b31a --- /dev/null +++ b/pagure_importer/utils/repo.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py + by pingou@pingoured.fr +''' + + +import pygit2 +import sys + + +def get_pygit2_version(): + ''' Return pygit2 version as a tuple of integers. + This is needed for correct version comparison. + ''' + return tuple([int(i) for i in pygit2.__version__.split('.')]) + + +class PagureRepo(pygit2.Repository): + """ An utility class allowing to go around pygit2's inability to be + stable. + + """ + + @staticmethod + def push(remote, refname): + """ Push the given reference to the specified remote. """ + pygit2_version = get_pygit2_version() + if pygit2_version >= (0, 22): + remote.push([refname]) + else: + remote.push(refname) + + def pull(self, remote_name='origin', branch='master', force=False): + ''' pull changes for the specified remote (defaults to origin). + + Code from MichaelBoselowitz at: + https://github.com/MichaelBoselowitz/pygit2-examples/blob/ + 68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58 + licensed under the MIT license. + ''' + + for remote in self.remotes: + if remote.name == remote_name: + remote.fetch() + remote_master_id = self.lookup_reference( + 'refs/remotes/origin/%s' % branch).target + + if force: + repo_branch = self.lookup_reference( + 'refs/heads/%s' % branch) + repo_branch.set_target(remote_master_id) + + merge_result, _ = self.merge_analysis(remote_master_id) + # Up to date, do nothing + if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: + return + # We can just fastforward + elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: + self.checkout_tree(self.get(remote_master_id)) + master_ref = self.lookup_reference( + 'refs/heads/%s' % branch) + master_ref.set_target(remote_master_id) + self.head.set_target(remote_master_id) + elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: + sys.exit('Pulling remote changes leads to a conflict') + else: + print 'Unexpected merge result: %s' % ( + pygit2.GIT_MERGE_ANALYSIS_NORMAL) + raise AssertionError('Unknown merge analysis result') diff --git a/pagure_importer/utils/trac.py b/pagure_importer/utils/trac.py new file mode 100644 index 0000000..d56a123 --- /dev/null +++ b/pagure_importer/utils/trac.py @@ -0,0 +1,90 @@ +from pagure_importer.utils.models import IssueComment, Issue +from datetime import datetime + + +def get_ticket_tags(trac_ticket): + return [] + + +def get_ticket_status(trac_ticket): + ''' Converts Trac ticket status + to Pagure issue status''' + + if trac_ticket['status'] != 'closed': + ticket_status = 'Open' + else: + ticket_status = 'Fixed' + return ticket_status + + +def populate_comments(fasclient, trac_comments): + comments = [] + for comment in trac_comments: + if comment[2] == 'comment' and comment[4] != '': + comment_user = comment[1] + pagure_issue_comment_user_email = None + pagure_issue_comment_body = comment[4] + pagure_issue_comment_created_at = datetime.strptime( + comment[0].value, "%Y%m%dT%H:%M:%S") + pagure_issue_comment_updated_at = None + + # No idea what to do with this right now + # editor: not supported by github api + pagure_issue_comment_parent = None + pagure_issue_comment_editor = None + + # comment updated at + pagure_issue_comment_edited_on = None + + # The User who commented + pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) + + # Object to represent comment on an issue + pagure_issue_comment = IssueComment( + id=None, + comment=pagure_issue_comment_body, + parent=pagure_issue_comment_parent, + date_created=pagure_issue_comment_created_at, + user=pagure_issue_comment_user.to_json(), + edited_on=pagure_issue_comment_edited_on, + editor=pagure_issue_comment_editor) + + comments.append(pagure_issue_comment.to_json()) + return comments + + +def populate_issue(trac, fasclient, ticket_id): + trac_ticket = trac.ticket.get(ticket_id)[3] + pagure_issue_title = trac_ticket['summary'] + pagure_issue_content = trac_ticket['description'] + + if pagure_issue_content == '': + pagure_issue_content = '#No Description Provided' + + pagure_issue_status = get_ticket_status(trac_ticket) + + pagure_issue_created_at = datetime.strptime( + trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") + + pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) + + pagure_issue_tags = get_ticket_tags(trac_ticket) + + pagure_issue_depends = [] + pagure_issue_blocks = [] + pagure_issue_is_private = False + + pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) + pagure_issue = Issue( + id=ticket_id, + title=pagure_issue_title, + content=pagure_issue_content, + status=pagure_issue_status, + date_created=pagure_issue_created_at, + user=pagure_issue_user.to_json(), + private=pagure_issue_is_private, + tags=pagure_issue_tags, + depends=pagure_issue_depends, + blocks=pagure_issue_blocks, + assignee=pagure_issue_assignee.to_json()) + return pagure_issue diff --git a/setup.py b/setup.py index e031405..65fa0d5 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( license='GNU General Public License v2.0', entry_points={ 'console_scripts': [ - 'pgimport = pagure_importer.run:main' + 'pgimport = pagure_importer.app:app' ], }, include_package_data=True, From d1338f49cdbc38864fe6f7f734dbfa0a39f50325 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 04 2016 19:22:34 +0000 Subject: [PATCH 17/24] Deal with non FAS user, for generic Trac instance --- diff --git a/pagure_importer/utils/trac.py b/pagure_importer/utils/trac.py index d56a123..4847959 100644 --- a/pagure_importer/utils/trac.py +++ b/pagure_importer/utils/trac.py @@ -1,4 +1,4 @@ -from pagure_importer.utils.models import IssueComment, Issue +from pagure_importer.utils.models import IssueComment, Issue, User from datetime import datetime @@ -37,7 +37,12 @@ def populate_comments(fasclient, trac_comments): pagure_issue_comment_edited_on = None # The User who commented - pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) + if fasclient: + pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) + else: + pagure_issue_comment_user = User(name='', + fullname='', + emails=[]) # Object to represent comment on an issue pagure_issue_comment = IssueComment( @@ -66,7 +71,13 @@ def populate_issue(trac, fasclient, ticket_id): pagure_issue_created_at = datetime.strptime( trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") - pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) + if fasclient: + pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) + pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) + else: + anonymous = User(name='', fullname='', emails=[]) + pagure_issue_assignee = anonymous + pagure_issue_user = anonymous pagure_issue_tags = get_ticket_tags(trac_ticket) @@ -74,7 +85,6 @@ def populate_issue(trac, fasclient, ticket_id): pagure_issue_blocks = [] pagure_issue_is_private = False - pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) pagure_issue = Issue( id=ticket_id, title=pagure_issue_title, From f0f28f906131a3e04c175786f815c2181f952c6f Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 04 2016 19:29:07 +0000 Subject: [PATCH 18/24] updated requirements.txt --- diff --git a/diff b/diff deleted file mode 100644 index 2a92e6e..0000000 --- a/diff +++ /dev/null @@ -1,1922 +0,0 @@ -diff --git a/pagure_importer/__init__.py b/pagure_importer/__init__.py -index fbab797..8b13789 100644 ---- a/pagure_importer/__init__.py -+++ b/pagure_importer/__init__.py -@@ -1,2 +1 @@ --import lib --import settings -+ -diff --git a/pagure_importer/app.py b/pagure_importer/app.py -new file mode 100644 -index 0000000..6dc3277 ---- /dev/null -+++ b/pagure_importer/app.py -@@ -0,0 +1,23 @@ -+#!/usr/bin/env python -+ -+import click -+import os -+ -+REPO_NAME = os.environ.get('REPO_NAME', None) # this has to be a bare repo -+REPO_PATH = os.environ.get('REPO_PATH', None) # the parent of the git directory -+ -+ -+@click.group() -+def app(): -+ pass -+ -+__all__ = [ -+ 'app', -+] -+ -+# from .commands import github -+from .commands import fedorahosted -+from .commands import github -+ -+if __name__ == '__main__': -+ app() -diff --git a/pagure_importer/commands/__init__.py b/pagure_importer/commands/__init__.py -new file mode 100644 -index 0000000..e69de29 -diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py -new file mode 100644 -index 0000000..650d133 ---- /dev/null -+++ b/pagure_importer/commands/fedorahosted.py -@@ -0,0 +1,15 @@ -+import click -+import getpass -+from pagure_importer.app import app, REPO_NAME, REPO_PATH -+from pagure_importer.utils import importer_trac -+from pagure_importer.utils.fas import FASclient -+ -+@app.command() -+@click.argument('project_url') -+def fedorahosted(project_url): -+ fas_username = raw_input('Enter you Github Username: ') -+ fas_password = getpass.getpass('Enter your github password: ') -+ fasclient = FASclient(fas_username, fas_password, -+ 'https://admin.fedoraproject.org/accounts') -+ trac_importer = importer_trac.TracImporter(project_url, fasclient) -+ trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) -diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py -new file mode 100644 -index 0000000..26ae4fa ---- /dev/null -+++ b/pagure_importer/commands/github.py -@@ -0,0 +1,40 @@ -+import click -+import getpass -+ -+from pagure_importer.app import app, REPO_NAME, REPO_PATH -+from pagure_importer.utils.importer_github import GithubImporter -+from pagure_importer.utils import ( -+ generate_json_for_github_contributors, -+ generate_json_for_github_issue_commentors, -+ assemble_github_contributors_commentors -+) -+ -+ -+def form_github_issues(): -+ github_username = raw_input('Enter you Github Username: ') -+ github_password = getpass.getpass('Enter your github password: ') -+ github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') -+ return (github_username, github_password, github_project_name) -+ -+@app.command() -+def github(): -+ github_username, github_password, github_project_name = form_github_issues() -+ gen_json = raw_input( -+ 'Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') -+ if gen_json == 'n': -+ github_importer = GithubImporter( -+ github_username=github_username, -+ github_password=github_password, -+ github_project_name=github_project_name) -+ github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) -+ else: -+ generate_json_for_github_contributors( -+ github_username, -+ github_password, -+ github_project_name) -+ generate_json_for_github_issue_commentors( -+ github_username, -+ github_password, -+ github_project_name) -+ assemble_github_contributors_commentors() -+ return -diff --git a/pagure_importer/forms.py b/pagure_importer/forms.py -deleted file mode 100644 -index a111114..0000000 ---- a/pagure_importer/forms.py -+++ /dev/null -@@ -1,10 +0,0 @@ --import getpass -- --from lib.sources.importer_github import GithubImporter --from settings import REPO_PATH, REPO_NAME -- --def form_github_issues(): -- github_username = raw_input('Enter you Github Username: ') -- github_password = getpass.getpass('Enter your github password: ') -- github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') -- return (github_username, github_password, github_project_name) -diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py -deleted file mode 100644 -index 9a64e3b..0000000 ---- a/pagure_importer/lib/__init__.py -+++ /dev/null -@@ -1,170 +0,0 @@ --import git --import models -- --import csv --import os --import getpass --import requests --import json --from github import Github --from requests.auth import HTTPBasicAuth --import pagure_importer --import pagure_importer.lib --from pagure_importer.lib.exceptions import FileNotFound, EmailNotFound -- --def generate_json_for_github_contributors(github_username, github_password, \ -- github_project_name): -- ''' Creates a file containing a list of dicts containing the username and -- emails of the contributors in the given github project -- ''' -- -- github_obj = Github(github_username, github_password) -- project = github_obj.get_repo(github_project_name) -- commits_url = project.commits_url.replace('{/sha}', '') -- -- page = 0 -- contributors = [] -- while True: -- page += 1 -- payload = {'page': page } -- data_ = json.loads(requests.get(commits_url, params=payload, -- auth=HTTPBasicAuth(github_username, github_password)).text) -- -- if not data_: -- break -- -- for data in data_: -- try: -- contributor = data['commit']['committer'] -- contributor_email = contributor['email'] -- contributor_fullname = contributor['name'] -- contributor_name = data['committer']['login'] -- except TypeError: -- print 'Maybe one of the contributors is dropped because of lack of details' -- continue -- -- json_data = { -- 'name': contributor_name, -- 'fullname': contributor_fullname, -- 'emails': [contributor_email] -- } -- -- present = False -- for i in contributors: -- if i == json_data: -- present = True -- break -- -- if not present: -- print 'contributor added: ', contributor_name -- contributors.append(json_data) -- -- with open('contributors.json', 'w') as f: -- f.write(json.dumps(contributors)) -- -- return -- -- --def generate_json_for_github_issue_commentors(github_username, github_password, \ -- github_project_name): -- ''' Will create a json file containing details of all the user -- who have commented on any issue in the given project -- ''' -- -- github_obj = Github(github_username, github_password) -- project = github_obj.get_repo(github_project_name) -- issue_comment_url = project.issue_comment_url.replace('{/number}', '') -- -- page = 0 -- issue_commentors = [] -- while True: -- page += 1 -- payload = {'page': page } -- data_ = json.loads(requests.get(issue_comment_url, params=payload, -- auth=HTTPBasicAuth(github_username, github_password)).text) -- -- if not data_: -- break -- -- for data in data_: -- try: -- commentor = data['user']['login'] -- except TypeError: -- print 'Maybe one of the issue commentors have been dropped because of lack of details' -- continue -- -- present = False -- for i in issue_commentors: -- if i == commentor: -- present = True -- break -- -- if not present: -- print 'commentor added: ', commentor -- issue_commentors.append(commentor) -- -- with open('issue_commentors.json', 'w') as f: -- f.write(json.dumps(issue_commentors)) -- return -- -- --def assemble_github_contributors_commentors(): -- ''' It uses the files: issue_commentors.json and contributors.json -- Assembles and creates a file: assembled_commentors.csv -- To use: just fill the empty blocks under emails column''' -- -- with open('issue_commentors.json', 'r') as ic: -- issue_names = json.load(ic) -- -- with open('contributors.json', 'r') as c: -- contributors = json.load(c) -- -- names = [] -- for i in issue_names: -- found = False -- for j in contributors: -- if j.get('name', None) == i: -- j['emails'] = j.get('emails')[0] -- names.append(j) -- found = True -- -- if not found: -- d = {'name': i, 'fullname': None, 'emails': None} -- names.append(d) -- -- with open('assembled_commentors.csv', 'w') as ac: -- field_names = ['name', 'fullname', 'emails'] -- writer = csv.DictWriter(ac, fieldnames=field_names) -- -- writer.writeheader() -- for name in names: -- writer.writerow(name) -- -- --def github_get_commentor_email(name): -- ''' Will return the issue commentor email as given in the -- assembled_commentors.csv file -- ''' -- -- if not os.path.exists('assembled_commentors.csv'): -- raise FileNotFound('The assembled_commentors.json file must be present \ -- Rerun the program and choose to generate the json files') -- -- data = [] -- with open('assembled_commentors.csv') as ac: -- reader = csv.DictReader(ac) -- for row in reader: -- data.append(dict( \ -- (('name', row['name']), \ -- ('fullname', row['fullname']), \ -- ('emails', row['emails'])))) -- -- -- for i in data: -- if i.get('name', None) == name: -- if i['emails']: -- return str(i['emails']) -- else: -- raise EmailNotFound('You need to fill out all the emails of the \ -- issue commentors') -- -diff --git a/pagure_importer/lib/exceptions.py b/pagure_importer/lib/exceptions.py -deleted file mode 100644 -index a5ab5fe..0000000 ---- a/pagure_importer/lib/exceptions.py -+++ /dev/null -@@ -1,22 +0,0 @@ --class GithubBadCredentials(Exception): -- ''' Raised when username/password for github is wrong ''' -- def __init__(self, msg): -- self.msg = msg -- -- --class GithubRepoNotFound(Exception): -- ''' Raised when the repo is not found for the user ''' -- def __init__(self, msg): -- self.msg = msg -- -- --class FileNotFound(Exception): -- ''' Raised when a certain file is not found ''' -- def __init__(self, msg): -- self.msg = msg -- -- --class EmailNotFound(Exception): -- ''' Raised when email is not found ''' -- def __init__(self, msg): -- self.msg = msg -diff --git a/pagure_importer/lib/fas.py b/pagure_importer/lib/fas.py -deleted file mode 100644 -index b63f665..0000000 ---- a/pagure_importer/lib/fas.py -+++ /dev/null -@@ -1,23 +0,0 @@ --from fedora.client.fas2 import AccountSystem --from pagure_importer.lib.models import User -- -- --class FASclient (): -- def __init__(self, fas_username, fas_password, fas_url): -- self.fasclient = AccountSystem(fas_url, username=fas_username, -- password=fas_password) -- -- anonymous = User(name='', fullname='', emails=[]) -- self.fasuser = {'': anonymous} -- -- def find_fas_user(self, user): -- -- if user not in self.fasuser.keys(): -- person = self.fasclient.person_by_username(user) -- if not person: -- return self.fasuser[''] -- -- self.fasuser[user] = User(name=user, -- fullname=person['human_name'], -- emails=[person['email']]) -- return self.fasuser[user] -diff --git a/pagure_importer/lib/git.py b/pagure_importer/lib/git.py -deleted file mode 100644 -index 33eba48..0000000 ---- a/pagure_importer/lib/git.py -+++ /dev/null -@@ -1,98 +0,0 @@ --''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py -- by pingou@pingoured.fr --''' -- --import shutil --import os --import pygit2 --import tempfile --import json -- --from repo import * -- --def update_git(obj, repo_path, repo_folder): -- """ Update the given issue in its git. -- This method forks the provided repo, add/edit the issue whose file name -- is defined by the uid field of the issue and if there are additions/ -- changes commit them and push them back to the original repo. -- """ -- -- if not repo_folder: -- return -- -- # Get the fork -- repopath = os.path.join(repo_folder, repo_path) -- -- # Clone the repo into a temp folder -- newpath = tempfile.mkdtemp(prefix='pagure-') -- new_repo = pygit2.clone_repository(repopath, newpath) -- -- file_path = os.path.join(newpath, obj.uid) -- -- # Get the current index -- index = new_repo.index -- -- # Are we adding files -- added = False -- if not os.path.exists(file_path): -- added = True -- -- # Write down what changed -- with open(file_path, 'w') as stream: -- stream.write(json.dumps( -- obj.to_json(), sort_keys=True, indent=4, -- separators=(',', ': '))) -- -- # Retrieve the list of files that changed -- diff = new_repo.diff() -- files = [] -- for p in diff: -- if hasattr(p, 'new_file_path'): -- files.append(p.new_file_path) -- elif hasattr(p, 'delta'): -- files.append(p.delta.new_file.path) -- -- # Add the changes to the index -- if added: -- index.add(obj.uid) -- for filename in files: -- index.add(filename) -- -- # If not change, return -- if not files and not added: -- shutil.rmtree(newpath) -- return -- -- # See if there is a parent to this commit -- parent = None -- try: -- parent = new_repo.head.get_object().oid -- except pygit2.GitError: -- pass -- -- parents = [] -- if parent: -- parents.append(parent) -- -- # Author/commiter will always be this one -- author = pygit2.Signature(name='pagure', email='pagure') -- -- # Actually commit -- new_repo.create_commit( -- 'refs/heads/master', -- author, -- author, -- 'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title), -- new_repo.index.write_tree(), -- parents) -- index.write() -- -- # Push to origin -- ori_remote = new_repo.remotes[0] -- master_ref = new_repo.lookup_reference('HEAD').resolve() -- refname = '%s:%s' % (master_ref.name, master_ref.name) -- -- PagureRepo.push(ori_remote, refname) -- -- # Remove the clone -- shutil.rmtree(newpath) -diff --git a/pagure_importer/lib/models.py b/pagure_importer/lib/models.py -deleted file mode 100644 -index 3aa3d83..0000000 ---- a/pagure_importer/lib/models.py -+++ /dev/null -@@ -1,105 +0,0 @@ --# -*- coding: utf-8 -*- -- --import datetime --import json --import uuid -- --class Issue(): -- ''' Represents an Issue ''' -- -- def __init__( -- self, id, title, content, -- status, date_created, user, private, tags, -- depends, blocks, assignee, comments=None): -- -- self.id = id -- self.title = title -- self.content = content -- self.status = status -- self.date_created = date_created -- self.user = user -- self.private = private -- self.tags = tags -- self.depends = depends -- self.blocks = blocks -- self.assignee = assignee -- self.comments = comments -- self.uid = uuid.uuid4().hex -- -- def to_json(self): -- ''' Returns a dictionary representation of the issue. -- -- ''' -- output = { -- 'id': self.id, -- 'title': self.title, -- 'content': self.content, -- 'status': self.status, -- 'date_created': self.date_created.strftime('%s'), -- 'user': self.user, -- 'private': self.private, -- 'tags': self.tags, -- 'depends': self.depends, -- 'blocks': self.blocks, -- 'assignee': self.assignee, -- 'comments': self.comments -- } -- -- return output -- -- @property -- def isa(self): -- return 'issue' -- -- --class IssueComment(): -- ''' Represent a comment for an issue ''' -- -- def __init__( -- self, id, comment, date_created, -- user, parent=None, edited_on=None, editor=None): -- -- self.id = id -- self.comment = comment -- self.parent = parent -- self.date_created = date_created -- self.user = user -- self.edited_on = edited_on -- self.editor = editor -- -- def to_json(self): -- ''' Returns a dictionary representation of the issue. ''' -- -- output = { -- 'id': self.id, -- 'comment': self.comment, -- 'parent': self.parent, -- 'date_created': self.date_created.strftime('%s'), -- 'user': self.user, -- 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, -- 'editor': self.editor or None -- } -- -- return output -- -- --class User(): -- ''' Represents a User ''' -- -- def __init__( -- self, name, emails, -- fullname=None): -- self.name = name -- self.fullname = fullname -- self.emails = emails -- -- def to_json(self): -- ''' Return a representation of the User in a dictionary. ''' -- -- output = { -- 'name': self.name, -- 'fullname': self.fullname, -- 'emails': self.emails -- } -- -- return output -diff --git a/pagure_importer/lib/repo.py b/pagure_importer/lib/repo.py -deleted file mode 100644 -index f46b31a..0000000 ---- a/pagure_importer/lib/repo.py -+++ /dev/null -@@ -1,70 +0,0 @@ --# -*- coding: utf-8 -*- -- --''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py -- by pingou@pingoured.fr --''' -- -- --import pygit2 --import sys -- -- --def get_pygit2_version(): -- ''' Return pygit2 version as a tuple of integers. -- This is needed for correct version comparison. -- ''' -- return tuple([int(i) for i in pygit2.__version__.split('.')]) -- -- --class PagureRepo(pygit2.Repository): -- """ An utility class allowing to go around pygit2's inability to be -- stable. -- -- """ -- -- @staticmethod -- def push(remote, refname): -- """ Push the given reference to the specified remote. """ -- pygit2_version = get_pygit2_version() -- if pygit2_version >= (0, 22): -- remote.push([refname]) -- else: -- remote.push(refname) -- -- def pull(self, remote_name='origin', branch='master', force=False): -- ''' pull changes for the specified remote (defaults to origin). -- -- Code from MichaelBoselowitz at: -- https://github.com/MichaelBoselowitz/pygit2-examples/blob/ -- 68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58 -- licensed under the MIT license. -- ''' -- -- for remote in self.remotes: -- if remote.name == remote_name: -- remote.fetch() -- remote_master_id = self.lookup_reference( -- 'refs/remotes/origin/%s' % branch).target -- -- if force: -- repo_branch = self.lookup_reference( -- 'refs/heads/%s' % branch) -- repo_branch.set_target(remote_master_id) -- -- merge_result, _ = self.merge_analysis(remote_master_id) -- # Up to date, do nothing -- if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: -- return -- # We can just fastforward -- elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: -- self.checkout_tree(self.get(remote_master_id)) -- master_ref = self.lookup_reference( -- 'refs/heads/%s' % branch) -- master_ref.set_target(remote_master_id) -- self.head.set_target(remote_master_id) -- elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: -- sys.exit('Pulling remote changes leads to a conflict') -- else: -- print 'Unexpected merge result: %s' % ( -- pygit2.GIT_MERGE_ANALYSIS_NORMAL) -- raise AssertionError('Unknown merge analysis result') -diff --git a/pagure_importer/lib/sources/__init__.py b/pagure_importer/lib/sources/__init__.py -deleted file mode 100644 -index e69de29..0000000 -diff --git a/pagure_importer/lib/sources/importer_github.py b/pagure_importer/lib/sources/importer_github.py -deleted file mode 100644 -index 2859e99..0000000 ---- a/pagure_importer/lib/sources/importer_github.py -+++ /dev/null -@@ -1,134 +0,0 @@ --from github import Github --import pagure_importer --import pagure_importer.lib --from pagure_importer.lib import models --from pagure_importer.lib import github_get_commentor_email --from pagure_importer.lib.exceptions import GithubBadCredentials, GithubRepoNotFound -- --class GithubImporter(): -- ''' Imports from Github using PyGithub and libpagure ''' -- def __init__( -- self, -- github_username, -- github_password, -- github_project_name): -- self.github_username = github_username -- self.github_password = github_password -- self.github_project_name = github_project_name -- self.github = Github(github_username, github_password) -- -- def import_issues(self, repo_path, repo_folder, status='all'): -- ''' Imports the issues on github for -- the given project -- ''' -- github_user = None -- try: -- github_user = self.github.get_user(self.github_username) -- except: -- raise GithubBadCredentials( -- 'Given github credentials are not correct') -- repo = self.github.get_repo(self.github_project_name) -- try: -- repo_name = repo.name -- except: -- raise GithubRepoNotFound( -- 'Repo not found, project name wrong') -- -- for github_issue in repo.get_issues(state=status): -- -- #title of the issue -- pagure_issue_title = github_issue.title -- -- #body of the issue -- if github_issue.body: -- pagure_issue_content = github_issue.body -- else: -- pagure_issue_content = '#No Description Provided' -- -- #Some details of a issue -- if github_issue.state != 'closed': -- pagure_issue_status = 'Open' -- else: -- pagure_issue_status = 'Fixed' -- -- pagure_issue_created_at = github_issue.created_at -- -- #Not sure how to deal with this atm -- pagure_issue_assignee = None -- -- if github_issue.labels: -- pagure_issue_tags = [i.name for i in github_issue.labels] -- else: -- pagure_issue_tags = [] -- -- -- #few things not supported by github -- pagure_issue_depends = [] -- pagure_issue_blocks = [] -- pagure_issue_is_private = False -- -- -- #User who created the issue -- pagure_issue_user = models.User( -- name=github_issue.user.login, -- fullname=github_issue.user.name, -- emails=[github_issue.user.email]) -- -- -- pagure_issue = models.Issue( -- id=None, -- title = pagure_issue_title, -- content = pagure_issue_content, -- status = pagure_issue_status, -- date_created = pagure_issue_created_at, -- user = pagure_issue_user.to_json(), -- private = pagure_issue_is_private, -- tags = pagure_issue_tags, -- depends = pagure_issue_depends, -- blocks = pagure_issue_blocks, -- assignee = pagure_issue_assignee) -- -- -- #comments on the issue -- comments = [] -- for comment in github_issue.get_comments(): -- -- comment_user = comment.user -- pagure_issue_comment_user_email = comment_user.email -- pagure_issue_comment_body = comment.body -- pagure_issue_comment_created_at = comment.created_at -- pagure_issue_comment_updated_at = comment.updated_at -- -- -- #No idea what to do with this right now -- #editor: not supported by github api -- pagure_issue_comment_parent = None -- pagure_issue_comment_editor = None -- -- #comment updated at -- pagure_issue_comment_edited_on = comment.updated_at -- -- #The User who commented -- pagure_issue_comment_user = models.User( -- name=comment_user.login, -- fullname=comment_user.name, -- emails=[comment_user.email] if comment_user.email \ -- else [github_get_commentor_email(comment_user.login)]) -- -- #Object to represent comment on an issue -- pagure_issue_comment = models.IssueComment( -- id=None, -- comment=pagure_issue_comment_body, -- parent=pagure_issue_comment_parent, -- date_created=pagure_issue_comment_created_at, -- user=pagure_issue_comment_user.to_json(), -- edited_on=pagure_issue_comment_edited_on, -- editor=pagure_issue_comment_editor) -- -- comments.append(pagure_issue_comment.to_json()) -- -- #add all the comments to the issue object -- pagure_issue.comments = comments -- -- #update the local git repo -- pagure_importer.lib.git.update_git(pagure_issue, repo_path, repo_folder) -diff --git a/pagure_importer/lib/sources/importer_trac.py b/pagure_importer/lib/sources/importer_trac.py -deleted file mode 100644 -index f69a3ce..0000000 ---- a/pagure_importer/lib/sources/importer_trac.py -+++ /dev/null -@@ -1,132 +0,0 @@ --from xmlrpclib import ServerProxy --import pagure_importer --import pagure_importer.lib --from pagure_importer.lib.fas import FASclient --from pagure_importer.lib import trac -- -- --class TracImporter(): -- '''Pagure importer for trac instance''' -- -- def __init__(self, trac_project_url): -- self.tracclient = ServerProxy(trac_project_url + '/rpc') -- self.fasclient = FASclient('user', 'password', -- 'https://admin.fedoraproject.org/accounts') -- -- def _find_fas_user(self, user): -- person = self.fasclient.person_by_username(user) -- human_name = person['human_name'] -- email = person['email'] -- pagure_user = models.User( -- name=user, -- fullname=human_name, -- emails=[email]) -- return pagure_user -- -- def _get_ticket_tags(self, trac_ticket): -- return [] -- -- def _get_ticket_status(self, trac_ticket): -- ''' Converts Trac ticket status -- to Pagure issue status''' -- -- if trac_ticket['status'] != 'closed': -- ticket_status = 'Open' -- else: -- ticket_status = 'Fixed' -- return ticket_status -- -- def _populate_comments(self, trac_comments): -- comments = [] -- for comment in trac_comments: -- if comment[2] == 'comment' and comment[4] != '': -- comment_user = comment[1] -- pagure_issue_comment_user_email = None -- pagure_issue_comment_body = comment[4] -- pagure_issue_comment_created_at = datetime.strptime( -- comment[0].value, "%Y%m%dT%H:%M:%S") -- pagure_issue_comment_updated_at = None -- -- # No idea what to do with this right now -- # editor: not supported by github api -- pagure_issue_comment_parent = None -- pagure_issue_comment_editor = None -- -- # comment updated at -- pagure_issue_comment_edited_on = None -- -- # The User who commented -- pagure_issue_comment_user = self._find_fas_user(comment[1]) -- -- # Object to represent comment on an issue -- pagure_issue_comment = models.IssueComment( -- id=None, -- comment=pagure_issue_comment_body, -- parent=pagure_issue_comment_parent, -- date_created=pagure_issue_comment_created_at, -- user=pagure_issue_comment_user.to_json(), -- edited_on=pagure_issue_comment_edited_on, -- editor=pagure_issue_comment_editor) -- -- comments.append(pagure_issue_comment.to_json()) -- return comments -- -- def _populate_issue(self, ticket_id): -- trac_ticket = self.trac.ticket.get(ticket_id)[3] -- pagure_issue_title = trac_ticket['summary'] -- pagure_issue_content = trac_ticket['description'] -- -- if pagure_issue_content == '': -- pagure_issue_content = '#No Description Provided' -- -- pagure_issue_status = self._get_ticket_status(trac_ticket) -- -- pagure_issue_created_at = datetime.strptime( -- self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") -- -- pagure_issue_assignee = trac_ticket['owner'] -- -- pagure_issue_tags = self._get_ticket_tags(trac_ticket) -- -- pagure_issue_depends = [] -- pagure_issue_blocks = [] -- pagure_issue_is_private = False -- -- pagure_issue_user = self._find_fas_user(trac_ticket['reporter']) -- -- pagure_issue = models.Issue( -- id=None, -- title=pagure_issue_title, -- content=pagure_issue_content, -- status=pagure_issue_status, -- date_created=pagure_issue_created_at, -- user=pagure_issue_user.to_json(), -- private=pagure_issue_is_private, -- tags=pagure_issue_tags, -- depends=pagure_issue_depends, -- blocks=pagure_issue_blocks, -- assignee=pagure_issue_assignee) -- return pagure_issue -- -- def import_issues(self, repo_path, repo_folder, -- trac_query='max=0&order=id'): -- '''Import issues from trac instance using xmlrpc API''' -- tickets_id = self.tracclient.ticket.query(trac_query) -- -- for ticket_id in tickets_id: -- -- pagure_issue = trac.populate_issue(self.tracclient, -- self.fasclient, ticket_id) -- -- pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) -- comments = trac.populate_comments(self.fasclient, -- pagure_issue_comments) -- -- # add all the comments to the issue object -- pagure_issue.comments = comments -- -- # update the local git repo -- print 'Update repo with issue :' + str(ticket_id) -- pagure_importer.lib.git.update_git(pagure_issue, -- repo_path, -- repo_folder) -diff --git a/pagure_importer/lib/trac.py b/pagure_importer/lib/trac.py -deleted file mode 100644 -index fb3d062..0000000 ---- a/pagure_importer/lib/trac.py -+++ /dev/null -@@ -1,90 +0,0 @@ --from pagure_importer.lib.models import IssueComment, Issue --from datetime import datetime -- -- --def get_ticket_tags(trac_ticket): -- return [] -- -- --def get_ticket_status(trac_ticket): -- ''' Converts Trac ticket status -- to Pagure issue status''' -- -- if trac_ticket['status'] != 'closed': -- ticket_status = 'Open' -- else: -- ticket_status = 'Fixed' -- return ticket_status -- -- --def populate_comments(fasclient, trac_comments): -- comments = [] -- for comment in trac_comments: -- if comment[2] == 'comment' and comment[4] != '': -- comment_user = comment[1] -- pagure_issue_comment_user_email = None -- pagure_issue_comment_body = comment[4] -- pagure_issue_comment_created_at = datetime.strptime( -- comment[0].value, "%Y%m%dT%H:%M:%S") -- pagure_issue_comment_updated_at = None -- -- # No idea what to do with this right now -- # editor: not supported by github api -- pagure_issue_comment_parent = None -- pagure_issue_comment_editor = None -- -- # comment updated at -- pagure_issue_comment_edited_on = None -- -- # The User who commented -- pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) -- -- # Object to represent comment on an issue -- pagure_issue_comment = IssueComment( -- id=None, -- comment=pagure_issue_comment_body, -- parent=pagure_issue_comment_parent, -- date_created=pagure_issue_comment_created_at, -- user=pagure_issue_comment_user.to_json(), -- edited_on=pagure_issue_comment_edited_on, -- editor=pagure_issue_comment_editor) -- -- comments.append(pagure_issue_comment.to_json()) -- return comments -- -- --def populate_issue(trac, fasclient, ticket_id): -- trac_ticket = trac.ticket.get(ticket_id)[3] -- pagure_issue_title = trac_ticket['summary'] -- pagure_issue_content = trac_ticket['description'] -- -- if pagure_issue_content == '': -- pagure_issue_content = '#No Description Provided' -- -- pagure_issue_status = get_ticket_status(trac_ticket) -- -- pagure_issue_created_at = datetime.strptime( -- trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") -- -- pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) -- -- pagure_issue_tags = get_ticket_tags(trac_ticket) -- -- pagure_issue_depends = [] -- pagure_issue_blocks = [] -- pagure_issue_is_private = False -- -- pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) -- pagure_issue = Issue( -- id=ticket_id, -- title=pagure_issue_title, -- content=pagure_issue_content, -- status=pagure_issue_status, -- date_created=pagure_issue_created_at, -- user=pagure_issue_user.to_json(), -- private=pagure_issue_is_private, -- tags=pagure_issue_tags, -- depends=pagure_issue_depends, -- blocks=pagure_issue_blocks, -- assignee=pagure_issue_assignee.to_json()) -- return pagure_issue -diff --git a/pagure_importer/run.py b/pagure_importer/run.py -deleted file mode 100644 -index d327228..0000000 ---- a/pagure_importer/run.py -+++ /dev/null -@@ -1,63 +0,0 @@ --#!/usr/bin/env python --import getpass --from forms import form_github_issues --from settings import IMPORT_SOURCES, IMPORT_OPTIONS, REPO_NAME, REPO_PATH --import pagure_importer --import pagure_importer.lib --import pagure_importer.lib.sources --from pagure_importer.lib.sources.importer_github import GithubImporter --from pagure_importer.lib.sources.importer_trac import TracImporter --from pagure_importer.lib import generate_json_for_github_contributors, \ -- generate_json_for_github_issue_commentors, \ -- assemble_github_contributors_commentors -- -- --def github_handler(item): -- if item.lower() == 'issues': -- github_username, github_password, github_project_name = form_github_issues() -- gen_json = raw_input('Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') -- if gen_json == 'n': -- github_importer = GithubImporter( -- github_username=github_username, -- github_password=github_password, -- github_project_name=github_project_name) -- github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) -- else: -- generate_json_for_github_contributors( -- github_username, -- github_password, -- github_project_name) -- generate_json_for_github_issue_commentors( -- github_username, -- github_password, -- github_project_name) -- assemble_github_contributors_commentors() -- return -- -- --def trac_handler(item, fedora=False): -- if item.lower() == 'issues': -- trac_url = raw_input('Enter the trac project url: ') -- trac_importer = TracImporter(trac_url) -- trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) -- -- --def main(): -- source = raw_input('Enter source from where you want to import: ') -- if source.lower() not in IMPORT_SOURCES: -- print 'Source location not supported' -- return -- -- item = raw_input('Enter the item to be imported: ') -- if item.lower() not in IMPORT_OPTIONS[source]: -- print 'Item import not supported' -- return -- -- if source.lower() == 'github': -- github_handler(item) -- elif source.lower() == 'fedorahosted': -- trac_handler(item, fedora=True) -- return -- --if __name__ == '__main__': -- main() -diff --git a/pagure_importer/settings.py b/pagure_importer/settings.py -deleted file mode 100644 -index 077ea40..0000000 ---- a/pagure_importer/settings.py -+++ /dev/null -@@ -1,7 +0,0 @@ --import os -- --IMPORT_SOURCES = ['github', 'fedorahosted'] --IMPORT_OPTIONS = {'github': ['issues'], 'fedorahosted': ['issues']} -- --REPO_NAME = os.environ.get('REPO_NAME', None) #this has to be a bare repo --REPO_PATH = os.environ.get('REPO_PATH', None) #the parent of the git directory -diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py -new file mode 100644 -index 0000000..05ccdb8 ---- /dev/null -+++ b/pagure_importer/utils/__init__.py -@@ -0,0 +1,168 @@ -+import git -+import models -+ -+import csv -+import os -+import getpass -+import requests -+import json -+from github import Github -+from requests.auth import HTTPBasicAuth -+ -+from pagure_importer.utils.exceptions import FileNotFound, EmailNotFound -+ -+def generate_json_for_github_contributors(github_username, github_password, \ -+ github_project_name): -+ ''' Creates a file containing a list of dicts containing the username and -+ emails of the contributors in the given github project -+ ''' -+ -+ github_obj = Github(github_username, github_password) -+ project = github_obj.get_repo(github_project_name) -+ commits_url = project.commits_url.replace('{/sha}', '') -+ -+ page = 0 -+ contributors = [] -+ while True: -+ page += 1 -+ payload = {'page': page } -+ data_ = json.loads(requests.get(commits_url, params=payload, -+ auth=HTTPBasicAuth(github_username, github_password)).text) -+ -+ if not data_: -+ break -+ -+ for data in data_: -+ try: -+ contributor = data['commit']['committer'] -+ contributor_email = contributor['email'] -+ contributor_fullname = contributor['name'] -+ contributor_name = data['committer']['login'] -+ except TypeError: -+ print 'Maybe one of the contributors is dropped because of lack of details' -+ continue -+ -+ json_data = { -+ 'name': contributor_name, -+ 'fullname': contributor_fullname, -+ 'emails': [contributor_email] -+ } -+ -+ present = False -+ for i in contributors: -+ if i == json_data: -+ present = True -+ break -+ -+ if not present: -+ print 'contributor added: ', contributor_name -+ contributors.append(json_data) -+ -+ with open('contributors.json', 'w') as f: -+ f.write(json.dumps(contributors)) -+ -+ return -+ -+ -+def generate_json_for_github_issue_commentors(github_username, github_password, \ -+ github_project_name): -+ ''' Will create a json file containing details of all the user -+ who have commented on any issue in the given project -+ ''' -+ -+ github_obj = Github(github_username, github_password) -+ project = github_obj.get_repo(github_project_name) -+ issue_comment_url = project.issue_comment_url.replace('{/number}', '') -+ -+ page = 0 -+ issue_commentors = [] -+ while True: -+ page += 1 -+ payload = {'page': page } -+ data_ = json.loads(requests.get(issue_comment_url, params=payload, -+ auth=HTTPBasicAuth(github_username, github_password)).text) -+ -+ if not data_: -+ break -+ -+ for data in data_: -+ try: -+ commentor = data['user']['login'] -+ except TypeError: -+ print 'Maybe one of the issue commentors have been dropped because of lack of details' -+ continue -+ -+ present = False -+ for i in issue_commentors: -+ if i == commentor: -+ present = True -+ break -+ -+ if not present: -+ print 'commentor added: ', commentor -+ issue_commentors.append(commentor) -+ -+ with open('issue_commentors.json', 'w') as f: -+ f.write(json.dumps(issue_commentors)) -+ return -+ -+ -+def assemble_github_contributors_commentors(): -+ ''' It uses the files: issue_commentors.json and contributors.json -+ Assembles and creates a file: assembled_commentors.csv -+ To use: just fill the empty blocks under emails column''' -+ -+ with open('issue_commentors.json', 'r') as ic: -+ issue_names = json.load(ic) -+ -+ with open('contributors.json', 'r') as c: -+ contributors = json.load(c) -+ -+ names = [] -+ for i in issue_names: -+ found = False -+ for j in contributors: -+ if j.get('name', None) == i: -+ j['emails'] = j.get('emails')[0] -+ names.append(j) -+ found = True -+ -+ if not found: -+ d = {'name': i, 'fullname': None, 'emails': None} -+ names.append(d) -+ -+ with open('assembled_commentors.csv', 'w') as ac: -+ field_names = ['name', 'fullname', 'emails'] -+ writer = csv.DictWriter(ac, fieldnames=field_names) -+ -+ writer.writeheader() -+ for name in names: -+ writer.writerow(name) -+ -+ -+def github_get_commentor_email(name): -+ ''' Will return the issue commentor email as given in the -+ assembled_commentors.csv file -+ ''' -+ -+ if not os.path.exists('assembled_commentors.csv'): -+ raise FileNotFound('The assembled_commentors.json file must be present \ -+ Rerun the program and choose to generate the json files') -+ -+ data = [] -+ with open('assembled_commentors.csv') as ac: -+ reader = csv.DictReader(ac) -+ for row in reader: -+ data.append(dict( \ -+ (('name', row['name']), \ -+ ('fullname', row['fullname']), \ -+ ('emails', row['emails'])))) -+ -+ -+ for i in data: -+ if i.get('name', None) == name: -+ if i['emails']: -+ return str(i['emails']) -+ else: -+ raise EmailNotFound('You need to fill out all the emails of the \ -+ issue commentors') -diff --git a/pagure_importer/utils/exceptions.py b/pagure_importer/utils/exceptions.py -new file mode 100644 -index 0000000..a5ab5fe ---- /dev/null -+++ b/pagure_importer/utils/exceptions.py -@@ -0,0 +1,22 @@ -+class GithubBadCredentials(Exception): -+ ''' Raised when username/password for github is wrong ''' -+ def __init__(self, msg): -+ self.msg = msg -+ -+ -+class GithubRepoNotFound(Exception): -+ ''' Raised when the repo is not found for the user ''' -+ def __init__(self, msg): -+ self.msg = msg -+ -+ -+class FileNotFound(Exception): -+ ''' Raised when a certain file is not found ''' -+ def __init__(self, msg): -+ self.msg = msg -+ -+ -+class EmailNotFound(Exception): -+ ''' Raised when email is not found ''' -+ def __init__(self, msg): -+ self.msg = msg -diff --git a/pagure_importer/utils/fas.py b/pagure_importer/utils/fas.py -new file mode 100644 -index 0000000..f62b5f5 ---- /dev/null -+++ b/pagure_importer/utils/fas.py -@@ -0,0 +1,23 @@ -+from fedora.client.fas2 import AccountSystem -+from pagure_importer.utils.models import User -+ -+ -+class FASclient (): -+ def __init__(self, fas_username, fas_password, fas_url): -+ self.fasclient = AccountSystem(fas_url, username=fas_username, -+ password=fas_password) -+ -+ anonymous = User(name='', fullname='', emails=[]) -+ self.fasuser = {'': anonymous} -+ -+ def find_fas_user(self, user): -+ -+ if user not in self.fasuser.keys(): -+ person = self.fasclient.person_by_username(user) -+ if not person: -+ return self.fasuser[''] -+ -+ self.fasuser[user] = User(name=user, -+ fullname=person['human_name'], -+ emails=[person['email']]) -+ return self.fasuser[user] -diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py -new file mode 100644 -index 0000000..33eba48 ---- /dev/null -+++ b/pagure_importer/utils/git.py -@@ -0,0 +1,98 @@ -+''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py -+ by pingou@pingoured.fr -+''' -+ -+import shutil -+import os -+import pygit2 -+import tempfile -+import json -+ -+from repo import * -+ -+def update_git(obj, repo_path, repo_folder): -+ """ Update the given issue in its git. -+ This method forks the provided repo, add/edit the issue whose file name -+ is defined by the uid field of the issue and if there are additions/ -+ changes commit them and push them back to the original repo. -+ """ -+ -+ if not repo_folder: -+ return -+ -+ # Get the fork -+ repopath = os.path.join(repo_folder, repo_path) -+ -+ # Clone the repo into a temp folder -+ newpath = tempfile.mkdtemp(prefix='pagure-') -+ new_repo = pygit2.clone_repository(repopath, newpath) -+ -+ file_path = os.path.join(newpath, obj.uid) -+ -+ # Get the current index -+ index = new_repo.index -+ -+ # Are we adding files -+ added = False -+ if not os.path.exists(file_path): -+ added = True -+ -+ # Write down what changed -+ with open(file_path, 'w') as stream: -+ stream.write(json.dumps( -+ obj.to_json(), sort_keys=True, indent=4, -+ separators=(',', ': '))) -+ -+ # Retrieve the list of files that changed -+ diff = new_repo.diff() -+ files = [] -+ for p in diff: -+ if hasattr(p, 'new_file_path'): -+ files.append(p.new_file_path) -+ elif hasattr(p, 'delta'): -+ files.append(p.delta.new_file.path) -+ -+ # Add the changes to the index -+ if added: -+ index.add(obj.uid) -+ for filename in files: -+ index.add(filename) -+ -+ # If not change, return -+ if not files and not added: -+ shutil.rmtree(newpath) -+ return -+ -+ # See if there is a parent to this commit -+ parent = None -+ try: -+ parent = new_repo.head.get_object().oid -+ except pygit2.GitError: -+ pass -+ -+ parents = [] -+ if parent: -+ parents.append(parent) -+ -+ # Author/commiter will always be this one -+ author = pygit2.Signature(name='pagure', email='pagure') -+ -+ # Actually commit -+ new_repo.create_commit( -+ 'refs/heads/master', -+ author, -+ author, -+ 'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title), -+ new_repo.index.write_tree(), -+ parents) -+ index.write() -+ -+ # Push to origin -+ ori_remote = new_repo.remotes[0] -+ master_ref = new_repo.lookup_reference('HEAD').resolve() -+ refname = '%s:%s' % (master_ref.name, master_ref.name) -+ -+ PagureRepo.push(ori_remote, refname) -+ -+ # Remove the clone -+ shutil.rmtree(newpath) -diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py -new file mode 100644 -index 0000000..cc61370 ---- /dev/null -+++ b/pagure_importer/utils/importer_github.py -@@ -0,0 +1,138 @@ -+from github import Github -+ -+from pagure_importer.utils import models -+from pagure_importer.utils import github_get_commentor_email -+from pagure_importer.utils.git import update_git -+from pagure_importer.utils.exceptions import ( -+ GithubBadCredentials, -+ GithubRepoNotFound -+) -+ -+ -+class GithubImporter(): -+ ''' Imports from Github using PyGithub and libpagure ''' -+ def __init__( -+ self, -+ github_username, -+ github_password, -+ github_project_name): -+ self.github_username = github_username -+ self.github_password = github_password -+ self.github_project_name = github_project_name -+ self.github = Github(github_username, github_password) -+ -+ def import_issues(self, repo_path, repo_folder, status='all'): -+ ''' Imports the issues on github for -+ the given project -+ ''' -+ github_user = None -+ try: -+ github_user = self.github.get_user(self.github_username) -+ except: -+ raise GithubBadCredentials( -+ 'Given github credentials are not correct') -+ repo = self.github.get_repo(self.github_project_name) -+ try: -+ repo_name = repo.name -+ except: -+ raise GithubRepoNotFound( -+ 'Repo not found, project name wrong') -+ -+ for github_issue in repo.get_issues(state=status): -+ -+ # title of the issue -+ pagure_issue_title = github_issue.title -+ -+ # body of the issue -+ if github_issue.body: -+ pagure_issue_content = github_issue.body -+ else: -+ pagure_issue_content = '#No Description Provided' -+ -+ # Some details of a issue -+ if github_issue.state != 'closed': -+ pagure_issue_status = 'Open' -+ else: -+ pagure_issue_status = 'Fixed' -+ -+ pagure_issue_created_at = github_issue.created_at -+ -+ # Not sure how to deal with this atm -+ pagure_issue_assignee = None -+ -+ if github_issue.labels: -+ pagure_issue_tags = [i.name for i in github_issue.labels] -+ else: -+ pagure_issue_tags = [] -+ -+ -+ # few things not supported by github -+ pagure_issue_depends = [] -+ pagure_issue_blocks = [] -+ pagure_issue_is_private = False -+ -+ -+ # User who created the issue -+ pagure_issue_user = models.User( -+ name=github_issue.user.login, -+ fullname=github_issue.user.name, -+ emails=[github_issue.user.email]) -+ -+ -+ pagure_issue = models.Issue( -+ id=None, -+ title = pagure_issue_title, -+ content = pagure_issue_content, -+ status = pagure_issue_status, -+ date_created = pagure_issue_created_at, -+ user = pagure_issue_user.to_json(), -+ private = pagure_issue_is_private, -+ tags = pagure_issue_tags, -+ depends = pagure_issue_depends, -+ blocks = pagure_issue_blocks, -+ assignee = pagure_issue_assignee) -+ -+ -+ # comments on the issue -+ comments = [] -+ for comment in github_issue.get_comments(): -+ -+ comment_user = comment.user -+ pagure_issue_comment_user_email = comment_user.email -+ pagure_issue_comment_body = comment.body -+ pagure_issue_comment_created_at = comment.created_at -+ pagure_issue_comment_updated_at = comment.updated_at -+ -+ -+ # No idea what to do with this right now -+ # editor: not supported by github api -+ pagure_issue_comment_parent = None -+ pagure_issue_comment_editor = None -+ -+ # comment updated at -+ pagure_issue_comment_edited_on = comment.updated_at -+ -+ # The User who commented -+ pagure_issue_comment_user = models.User( -+ name=comment_user.login, -+ fullname=comment_user.name, -+ emails=[comment_user.email] if comment_user.email \ -+ else [github_get_commentor_email(comment_user.login)]) -+ -+ # Object to represent comment on an issue -+ pagure_issue_comment = models.IssueComment( -+ id=None, -+ comment=pagure_issue_comment_body, -+ parent=pagure_issue_comment_parent, -+ date_created=pagure_issue_comment_created_at, -+ user=pagure_issue_comment_user.to_json(), -+ edited_on=pagure_issue_comment_edited_on, -+ editor=pagure_issue_comment_editor) -+ -+ comments.append(pagure_issue_comment.to_json()) -+ -+ # add all the comments to the issue object -+ pagure_issue.comments = comments -+ -+ # update the local git repo -+ update_git(pagure_issue, repo_path, repo_folder) -diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py -new file mode 100644 -index 0000000..50f20b0 ---- /dev/null -+++ b/pagure_importer/utils/importer_trac.py -@@ -0,0 +1,35 @@ -+from xmlrpclib import ServerProxy -+from pagure_importer.utils.git import update_git -+from pagure_importer.utils import trac -+ -+ -+class TracImporter(): -+ '''Pagure importer for trac instance''' -+ -+ def __init__(self, trac_project_url, fasclient=None): -+ self.tracclient = ServerProxy(trac_project_url + '/rpc') -+ if fasclient: -+ self.fasclient = fasclient -+ -+ def import_issues(self, repo_path, repo_folder, -+ trac_query='max=0&order=id'): -+ '''Import issues from trac instance using xmlrpc API''' -+ tickets_id = self.tracclient.ticket.query(trac_query) -+ -+ for ticket_id in tickets_id: -+ -+ pagure_issue = trac.populate_issue(self.tracclient, -+ self.fasclient, ticket_id) -+ -+ pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) -+ comments = trac.populate_comments(self.fasclient, -+ pagure_issue_comments) -+ -+ # add all the comments to the issue object -+ pagure_issue.comments = comments -+ -+ # update the local git repo -+ print 'Update repo with issue :' + str(ticket_id) -+ update_git(pagure_issue, -+ repo_path, -+ repo_folder) -diff --git a/pagure_importer/utils/models.py b/pagure_importer/utils/models.py -new file mode 100644 -index 0000000..3aa3d83 ---- /dev/null -+++ b/pagure_importer/utils/models.py -@@ -0,0 +1,105 @@ -+# -*- coding: utf-8 -*- -+ -+import datetime -+import json -+import uuid -+ -+class Issue(): -+ ''' Represents an Issue ''' -+ -+ def __init__( -+ self, id, title, content, -+ status, date_created, user, private, tags, -+ depends, blocks, assignee, comments=None): -+ -+ self.id = id -+ self.title = title -+ self.content = content -+ self.status = status -+ self.date_created = date_created -+ self.user = user -+ self.private = private -+ self.tags = tags -+ self.depends = depends -+ self.blocks = blocks -+ self.assignee = assignee -+ self.comments = comments -+ self.uid = uuid.uuid4().hex -+ -+ def to_json(self): -+ ''' Returns a dictionary representation of the issue. -+ -+ ''' -+ output = { -+ 'id': self.id, -+ 'title': self.title, -+ 'content': self.content, -+ 'status': self.status, -+ 'date_created': self.date_created.strftime('%s'), -+ 'user': self.user, -+ 'private': self.private, -+ 'tags': self.tags, -+ 'depends': self.depends, -+ 'blocks': self.blocks, -+ 'assignee': self.assignee, -+ 'comments': self.comments -+ } -+ -+ return output -+ -+ @property -+ def isa(self): -+ return 'issue' -+ -+ -+class IssueComment(): -+ ''' Represent a comment for an issue ''' -+ -+ def __init__( -+ self, id, comment, date_created, -+ user, parent=None, edited_on=None, editor=None): -+ -+ self.id = id -+ self.comment = comment -+ self.parent = parent -+ self.date_created = date_created -+ self.user = user -+ self.edited_on = edited_on -+ self.editor = editor -+ -+ def to_json(self): -+ ''' Returns a dictionary representation of the issue. ''' -+ -+ output = { -+ 'id': self.id, -+ 'comment': self.comment, -+ 'parent': self.parent, -+ 'date_created': self.date_created.strftime('%s'), -+ 'user': self.user, -+ 'edited_on': self.edited_on.strftime('%s') if self.edited_on else None, -+ 'editor': self.editor or None -+ } -+ -+ return output -+ -+ -+class User(): -+ ''' Represents a User ''' -+ -+ def __init__( -+ self, name, emails, -+ fullname=None): -+ self.name = name -+ self.fullname = fullname -+ self.emails = emails -+ -+ def to_json(self): -+ ''' Return a representation of the User in a dictionary. ''' -+ -+ output = { -+ 'name': self.name, -+ 'fullname': self.fullname, -+ 'emails': self.emails -+ } -+ -+ return output -diff --git a/pagure_importer/utils/repo.py b/pagure_importer/utils/repo.py -new file mode 100644 -index 0000000..f46b31a ---- /dev/null -+++ b/pagure_importer/utils/repo.py -@@ -0,0 +1,70 @@ -+# -*- coding: utf-8 -*- -+ -+''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py -+ by pingou@pingoured.fr -+''' -+ -+ -+import pygit2 -+import sys -+ -+ -+def get_pygit2_version(): -+ ''' Return pygit2 version as a tuple of integers. -+ This is needed for correct version comparison. -+ ''' -+ return tuple([int(i) for i in pygit2.__version__.split('.')]) -+ -+ -+class PagureRepo(pygit2.Repository): -+ """ An utility class allowing to go around pygit2's inability to be -+ stable. -+ -+ """ -+ -+ @staticmethod -+ def push(remote, refname): -+ """ Push the given reference to the specified remote. """ -+ pygit2_version = get_pygit2_version() -+ if pygit2_version >= (0, 22): -+ remote.push([refname]) -+ else: -+ remote.push(refname) -+ -+ def pull(self, remote_name='origin', branch='master', force=False): -+ ''' pull changes for the specified remote (defaults to origin). -+ -+ Code from MichaelBoselowitz at: -+ https://github.com/MichaelBoselowitz/pygit2-examples/blob/ -+ 68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58 -+ licensed under the MIT license. -+ ''' -+ -+ for remote in self.remotes: -+ if remote.name == remote_name: -+ remote.fetch() -+ remote_master_id = self.lookup_reference( -+ 'refs/remotes/origin/%s' % branch).target -+ -+ if force: -+ repo_branch = self.lookup_reference( -+ 'refs/heads/%s' % branch) -+ repo_branch.set_target(remote_master_id) -+ -+ merge_result, _ = self.merge_analysis(remote_master_id) -+ # Up to date, do nothing -+ if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: -+ return -+ # We can just fastforward -+ elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: -+ self.checkout_tree(self.get(remote_master_id)) -+ master_ref = self.lookup_reference( -+ 'refs/heads/%s' % branch) -+ master_ref.set_target(remote_master_id) -+ self.head.set_target(remote_master_id) -+ elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: -+ sys.exit('Pulling remote changes leads to a conflict') -+ else: -+ print 'Unexpected merge result: %s' % ( -+ pygit2.GIT_MERGE_ANALYSIS_NORMAL) -+ raise AssertionError('Unknown merge analysis result') -diff --git a/pagure_importer/utils/trac.py b/pagure_importer/utils/trac.py -new file mode 100644 -index 0000000..d56a123 ---- /dev/null -+++ b/pagure_importer/utils/trac.py -@@ -0,0 +1,90 @@ -+from pagure_importer.utils.models import IssueComment, Issue -+from datetime import datetime -+ -+ -+def get_ticket_tags(trac_ticket): -+ return [] -+ -+ -+def get_ticket_status(trac_ticket): -+ ''' Converts Trac ticket status -+ to Pagure issue status''' -+ -+ if trac_ticket['status'] != 'closed': -+ ticket_status = 'Open' -+ else: -+ ticket_status = 'Fixed' -+ return ticket_status -+ -+ -+def populate_comments(fasclient, trac_comments): -+ comments = [] -+ for comment in trac_comments: -+ if comment[2] == 'comment' and comment[4] != '': -+ comment_user = comment[1] -+ pagure_issue_comment_user_email = None -+ pagure_issue_comment_body = comment[4] -+ pagure_issue_comment_created_at = datetime.strptime( -+ comment[0].value, "%Y%m%dT%H:%M:%S") -+ pagure_issue_comment_updated_at = None -+ -+ # No idea what to do with this right now -+ # editor: not supported by github api -+ pagure_issue_comment_parent = None -+ pagure_issue_comment_editor = None -+ -+ # comment updated at -+ pagure_issue_comment_edited_on = None -+ -+ # The User who commented -+ pagure_issue_comment_user = fasclient.find_fas_user(comment[1]) -+ -+ # Object to represent comment on an issue -+ pagure_issue_comment = IssueComment( -+ id=None, -+ comment=pagure_issue_comment_body, -+ parent=pagure_issue_comment_parent, -+ date_created=pagure_issue_comment_created_at, -+ user=pagure_issue_comment_user.to_json(), -+ edited_on=pagure_issue_comment_edited_on, -+ editor=pagure_issue_comment_editor) -+ -+ comments.append(pagure_issue_comment.to_json()) -+ return comments -+ -+ -+def populate_issue(trac, fasclient, ticket_id): -+ trac_ticket = trac.ticket.get(ticket_id)[3] -+ pagure_issue_title = trac_ticket['summary'] -+ pagure_issue_content = trac_ticket['description'] -+ -+ if pagure_issue_content == '': -+ pagure_issue_content = '#No Description Provided' -+ -+ pagure_issue_status = get_ticket_status(trac_ticket) -+ -+ pagure_issue_created_at = datetime.strptime( -+ trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S") -+ -+ pagure_issue_assignee = fasclient.find_fas_user(trac_ticket['owner']) -+ -+ pagure_issue_tags = get_ticket_tags(trac_ticket) -+ -+ pagure_issue_depends = [] -+ pagure_issue_blocks = [] -+ pagure_issue_is_private = False -+ -+ pagure_issue_user = fasclient.find_fas_user(trac_ticket['reporter']) -+ pagure_issue = Issue( -+ id=ticket_id, -+ title=pagure_issue_title, -+ content=pagure_issue_content, -+ status=pagure_issue_status, -+ date_created=pagure_issue_created_at, -+ user=pagure_issue_user.to_json(), -+ private=pagure_issue_is_private, -+ tags=pagure_issue_tags, -+ depends=pagure_issue_depends, -+ blocks=pagure_issue_blocks, -+ assignee=pagure_issue_assignee.to_json()) -+ return pagure_issue -diff --git a/setup.py b/setup.py -index e031405..65fa0d5 100644 ---- a/setup.py -+++ b/setup.py -@@ -32,7 +32,7 @@ setup( - license='GNU General Public License v2.0', - entry_points={ - 'console_scripts': [ -- 'pgimport = pagure_importer.run:main' -+ 'pgimport = pagure_importer.app:app' - ], - }, - include_package_data=True, diff --git a/requirements.txt b/requirements.txt index 52fc450..e966d85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,5 @@ PyGithub requests +click +python-fedora +pygit2 >= 0.20.1 From 98445b30c3794779bf1b5404bdc265022e3e006e Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 04 2016 20:33:19 +0000 Subject: [PATCH 19/24] Added usage to README + fix requirements.txt --- diff --git a/README.md b/README.md index b2075e2..0d3d02b 100644 --- a/README.md +++ b/README.md @@ -2,25 +2,54 @@ CLI tool for importing issues etc. from different sources like github to pagure ## Installation +--- 1. Install it using ```pip``` . ```pip install pagure_importer``` ## How to run +--- 0. Clone the issue tracker for issues from pagure. Use: ```git clone --bare``` 1. set the env variables: ```REPO_NAME``` and ```REPO_PATH``` ex: REPO_NAME=abc.git; REPO_PATH=/home/vivek/ 2. Activate the pagure tickets hook from project settings. -3. Execute ```pgimport``` +3. Execute ```pgimport```. See Usage section 4. Just answer what is asked. Check below instructions for particular source 5. The script will make commits in your cloned bare repo: push the changes back to pagure. -### Present options for sources: github -### Present options for items: issues + +## Usage +----- + + + $ pgimport --help + Usage: pgimport [OPTIONS] COMMAND [ARGS]... + + Options: + --help Show this message and exit. + + Commands: + fedorahosted + github + + +The fedorahosted command can be used to import issues from a fedorahosted project to pagure + + $ pgimport fedorahosted https://fedorahosted.org/fedocal + + +The github command can be used to import issues from a github project to pagure + + $ pgimport github + ### Tools used: +--- 1. [PyGithub](https://github.com/PyGithub/PyGithub) - a python library for [github](https://github.com/) api. +2. [click](https://github.com/pallets/click) - Python package for creating beautiful command line interfaces +3. [python-fedora](https://fedorahosted.org/python-fedora/) - A collection of python code that allows programs to talk to Fedora Services ## How it works: Github Issues +--- 0. For github issues, there is a bit of pre-processing so, the process is not very user friendly. The reason behind the pre-processing is that: github doesn't give away the email ids of issue commentors unless the commentor diff --git a/requirements.txt b/requirements.txt index e966d85..ba8d9e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ PyGithub -requests click python-fedora pygit2 >= 0.20.1 From c76ad225d967731ebee6d3f7b4d50b225a43a6c6 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 08 2016 08:25:32 +0000 Subject: [PATCH 20/24] removed debug comment + fix README separation line --- diff --git a/README.md b/README.md index 0d3d02b..4e8c209 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ ex: REPO_NAME=abc.git; REPO_PATH=/home/vivek/ ## Usage ------ +--- $ pgimport --help diff --git a/pagure_importer/app.py b/pagure_importer/app.py index 6dc3277..7c92a53 100644 --- a/pagure_importer/app.py +++ b/pagure_importer/app.py @@ -15,7 +15,6 @@ __all__ = [ 'app', ] -# from .commands import github from .commands import fedorahosted from .commands import github From e0c21e9c98dac76c4ca7a81fb5a81c5523bd0726 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 09 2016 17:00:47 +0000 Subject: [PATCH 21/24] Display the number of issue to imported --- diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index 8e7fff8..9fdcd14 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -7,7 +7,7 @@ class TracImporter(): '''Pagure importer for trac instance''' def __init__(self, trac_project_url, fasclient=None): - self.tracclient = ServerProxy(trac_project_url + '/rpc') + self.tracclient = ServerProxy(trac_project_url) self.fasclient = fasclient def import_issues(self, repo_path, repo_folder, @@ -28,5 +28,6 @@ class TracImporter(): pagure_issue.comments = comments # update the local git repo - print 'Update repo with issue :' + str(ticket_id) + print 'Update repo with issue :' + str(ticket_id) + '/' +\ + str(tickets_id[-1]) update_git(pagure_issue, repo_path, repo_folder) From 8fd92641c46e8b28f549627f5417743a7e5867c4 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 09 2016 17:04:59 +0000 Subject: [PATCH 22/24] Use FAS username and password to to perform authenticated calls of the XML-RPC APIs --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 9ca8a53..15fc1c4 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -11,5 +11,10 @@ def fedorahosted(project_url): fas_password = getpass.getpass('Enter your FAS password: ') fasclient = FASclient(fas_username, fas_password, 'https://admin.fedoraproject.org/accounts') - trac_importer = importer_trac.TracImporter(project_url, fasclient) + + rpc_login = fas_username + ':' + fas_password + '@' + url_index = project_url.find('://') + rpc_url = project_url[:url_index+3] + rpc_login + project_url[url_index+3:]\ + + '/login/xmlrpc' + trac_importer = importer_trac.TracImporter(rpc_url, fasclient) trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) From 4a6b28b480a90ce7b9a4a46517a1710cf1987311 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 10 2016 22:20:03 +0000 Subject: [PATCH 23/24] Added basic support for issue tags --- diff --git a/pagure_importer/utils/trac.py b/pagure_importer/utils/trac.py index 4847959..5abb015 100644 --- a/pagure_importer/utils/trac.py +++ b/pagure_importer/utils/trac.py @@ -2,10 +2,6 @@ from pagure_importer.utils.models import IssueComment, Issue, User from datetime import datetime -def get_ticket_tags(trac_ticket): - return [] - - def get_ticket_status(trac_ticket): ''' Converts Trac ticket status to Pagure issue status''' @@ -79,7 +75,15 @@ def populate_issue(trac, fasclient, ticket_id): pagure_issue_assignee = anonymous pagure_issue_user = anonymous - pagure_issue_tags = get_ticket_tags(trac_ticket) + pagure_issue_tags = [] + if trac_ticket['type'] != '': + pagure_issue_tags.append(trac_ticket['type']) + if trac_ticket['milestone'] != '': + pagure_issue_tags.append(trac_ticket['milestone']) + if trac_ticket['component'] != '': + pagure_issue_tags.append(trac_ticket['component']) + if trac_ticket['version'] != '': + pagure_issue_tags.append(trac_ticket['version']) pagure_issue_depends = [] pagure_issue_blocks = [] From 9af8efeab8e3962da81356b3aa16d2009caf4740 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 13 2016 19:45:06 +0000 Subject: [PATCH 24/24] Added Import pagure tags option to fedorahosted command If flag is set then we import the tags, otherwise tags will be left empty --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 15fc1c4..403d269 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -6,7 +6,8 @@ from pagure_importer.utils.fas import FASclient @app.command() @click.argument('project_url') -def fedorahosted(project_url): +@click.option('--tags', help="Import pagure tags:", is_flag=True) +def fedorahosted(project_url, tags): fas_username = raw_input('Enter you FAS Username: ') fas_password = getpass.getpass('Enter your FAS password: ') fasclient = FASclient(fas_username, fas_password, @@ -17,4 +18,5 @@ def fedorahosted(project_url): rpc_url = project_url[:url_index+3] + rpc_login + project_url[url_index+3:]\ + '/login/xmlrpc' trac_importer = importer_trac.TracImporter(rpc_url, fasclient) - trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) + trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH, + tags=tags) diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index 9fdcd14..deb2cb8 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -10,7 +10,7 @@ class TracImporter(): self.tracclient = ServerProxy(trac_project_url) self.fasclient = fasclient - def import_issues(self, repo_path, repo_folder, + def import_issues(self, repo_path, repo_folder, tags, trac_query='max=0&order=id'): '''Import issues from trac instance using xmlrpc API''' tickets_id = self.tracclient.ticket.query(trac_query) @@ -18,7 +18,7 @@ class TracImporter(): for ticket_id in tickets_id: pagure_issue = trac.populate_issue(self.tracclient, - self.fasclient, ticket_id) + self.fasclient, ticket_id, tags) pagure_issue_comments = self.tracclient.ticket.changeLog(ticket_id) comments = trac.populate_comments(self.fasclient, diff --git a/pagure_importer/utils/trac.py b/pagure_importer/utils/trac.py index 5abb015..b5e131b 100644 --- a/pagure_importer/utils/trac.py +++ b/pagure_importer/utils/trac.py @@ -54,7 +54,7 @@ def populate_comments(fasclient, trac_comments): return comments -def populate_issue(trac, fasclient, ticket_id): +def populate_issue(trac, fasclient, ticket_id, tags): trac_ticket = trac.ticket.get(ticket_id)[3] pagure_issue_title = trac_ticket['summary'] pagure_issue_content = trac_ticket['description'] @@ -76,14 +76,15 @@ def populate_issue(trac, fasclient, ticket_id): pagure_issue_user = anonymous pagure_issue_tags = [] - if trac_ticket['type'] != '': - pagure_issue_tags.append(trac_ticket['type']) - if trac_ticket['milestone'] != '': - pagure_issue_tags.append(trac_ticket['milestone']) - if trac_ticket['component'] != '': - pagure_issue_tags.append(trac_ticket['component']) - if trac_ticket['version'] != '': - pagure_issue_tags.append(trac_ticket['version']) + if tags: + if trac_ticket['type'] != '': + pagure_issue_tags.append(trac_ticket['type']) + if trac_ticket['milestone'] != '': + pagure_issue_tags.append(trac_ticket['milestone']) + if trac_ticket['component'] != '': + pagure_issue_tags.append(trac_ticket['component']) + if trac_ticket['version'] != '': + pagure_issue_tags.append(trac_ticket['version']) pagure_issue_depends = [] pagure_issue_blocks = []