From f83a16720bc3c9241cade1e4a7b2d17ae429705d Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jan 03 2017 16:21:25 +0000
Subject: [PATCH 1/13] Make to_timestamp a function


to_timestamp has nothing to do with the TracImporter class it was on, the proof is that
it never used the ``self`` argument passed. So this commit moves it to a function instead
of it being a method of TracImporter.

Eventually, it could even be moved into another module as it is also not specific to trac
and therefore, could eventually be used by other pieces of code.

---

diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index 63ae0a6..a403836 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -12,6 +12,15 @@ from pagure_importer.utils.git import (
 from pagure_importer.utils.models import User, Issue, IssueComment
 
 
+def to_timestamp(tm):
+    ''' Convert to timestamp which can be jsonified '''
+
+    tm = tm.replace('+00:00', '')
+    date = datetime.strptime(tm, '%Y-%m-%dT%H:%M:%S')
+    ts = str(time.mktime(date.timetuple()))[:-2]  # Strip the .0
+    return ts
+
+
 class TracImporter(Importer):
     ''' Pagure importer for trac instance '''
 
@@ -52,13 +61,6 @@ class TracImporter(Importer):
 
         return resp['result']
 
-    def to_timestamp(self, tm):
-        ''' Convert to timestamp which can be jsonified '''
-
-        tm = tm.replace('+00:00', '')
-        date = datetime.strptime(tm, '%Y-%m-%dT%H:%M:%S')
-        ts = str(time.mktime(date.timetuple()))[:-2]  # Strip the .0
-        return ts
 
     def get_custom_fields(self):
         ''' Queries the fedorahosted api to get all ticket fields
@@ -159,7 +161,7 @@ class TracImporter(Importer):
 
         issue_status, close_status = self.get_ticket_status(trac_ticket)
 
-        pagure_issue_created_at = self.to_timestamp(
+        pagure_issue_created_at = to_timestamp(
             trac_ticket_info[1]['__jsonclass__'][1])
 
         if self.fas:
@@ -259,7 +261,7 @@ class TracImporter(Importer):
 
         comments = {}
         for comment in trac_comments:
-            ts = self.to_timestamp(comment[0]['__jsonclass__'][1])
+            ts = to_timestamp(comment[0]['__jsonclass__'][1])
 
             if comment[2] == 'comment' and comment[4] != '':
                 if ts in comments:

From 89bd4fb649179b63a5713c7fbabe8cfa51b7765e Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jan 03 2017 17:30:09 +0000
Subject: [PATCH 2/13] Refactor the logic in import_issues for trac


Currently when calling import issues the program does several things:
- Clone the git repo or rather lists all cloned repo and asks which one
should be used
- Queries all the tickets in trac
- Convert them to JSON blob
- Import them into the git repo selected in the first step
- Push said git repo and delete it from the filesystem

With this commits, the logic is moved into the CLI itself, which now
does select where is the git repo to use, then does the querying of trac
and converting to JSON, does the import into git in one step at the end
speeding up the process and finally push and clean as before.

This means that import_issues() can now be used outside of any git repo
and it will just output the JSON blob in the specified directory.

---

diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py
index ee87533..9164b6c 100644
--- a/pagure_importer/commands/fedorahosted.py
+++ b/pagure_importer/commands/fedorahosted.py
@@ -1,9 +1,11 @@
 import click
 import pagure_importer
 from pagure_importer.app import app, REPO_PATH
-from pagure_importer.utils import importer_trac
+from pagure_importer.utils import importer_trac, get_pagure_namespace
 from pagure_importer.utils.fas import FASclient
 
+import pagure_importer.utils.git as gitutils
+
 
 @app.command()
 @click.argument('project_url')
@@ -26,6 +28,8 @@ def fedorahosted(project_url, tags, private, username, password, offset):
         repo_index = click.prompt('Choose the import destination repo ',
                                   default=1)
         repo_name = repos[int(repo_index)-1]
+        newpath, new_repo = gitutils.clone_repo(repo_name, REPO_PATH)
+        project = get_pagure_namespace(REPO_PATH, repo_name)
         with importer_trac.TracImporter(project_url=project_url,
                                         username=username,
                                         password=password,
@@ -36,6 +40,9 @@ def fedorahosted(project_url, tags, private, username, password, offset):
                                         tags=tags,
                                         private=private) as trac_importer:
 
-            trac_importer.import_issues()
+            trac_importer.import_issues(project, REPO_PATH)
+        # update the local git repo
+        new_repo = gitutils.update_git(newpath, new_repo)
+        gitutils.push_delete_repo(newpath, new_repo)
     else:
         click.echo('No ticket repository found. Use pgimport clone command')
diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py
index b38935b..52fc241 100644
--- a/pagure_importer/utils/__init__.py
+++ b/pagure_importer/utils/__init__.py
@@ -226,8 +226,7 @@ def get_pagure_namespace(repo_folder, repo_name):
     remote_path = urlparse(remote_url).path
     remote_path = remote_path.replace('.git', '')
     namespace_list = remote_path.split('/')[2:]
-    namespace = '/'.join(namespace_list)
-    return namespace
+    return '/'.join(namespace_list)
 
 
 def is_image(filename):
@@ -248,10 +247,50 @@ class Importer:
         self.password = password
         self.repo_name = repo_name
         self.repo_folder = repo_folder
+        self.clone_repo_location = os.path.join(
+            repo_folder, 'clone-' + repo_name)
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         ''' Delete the cloned repo where the commits were going '''
-        shutil.rmtree(os.path.join(self.repo_folder, 'clone-' + self.repo_name))
+        if os.path.exists(clone_repo_location):
+            shutil.rmtree(clone_repo_location)
+
+
+def issue_to_json(issue, folder):
+    ''' Write the specified issue as a JSON blob on the specified folder.
+    Returns a list of all the files changed or created.
+
+    :arg issue: a
+
+    '''
+    file_path = os.path.join(folder, issue.uid)
+    files = []
+
+    # Are we adding files
+    added = False
+    if not os.path.exists(file_path):
+        files.append(issue.uid)
+
+    # If we have attachments
+    attachments = issue.attachment
+    if attachments:
+        if not os.path.exists(os.path.join(newpath, 'files')):
+            os.mkdir(os.path.join(newpath, 'files'))
+
+        for key in attachments.keys():
+            filename = get_secure_filename(attachments[key], key)
+            attach_path = os.path.join(newpath, 'files', filename)
+            with open(attach_path, 'w') as stream:
+                stream.write(str(attachments[key]))
+            files.append('files/' + filename)
+
+    # Write down what changed
+    with open(file_path, 'w') as stream:
+        stream.write(json.dumps(
+            issue.to_json(), sort_keys=True, indent=4,
+            separators=(',', ': ')))
+
+    return files
diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py
index ff51324..7bee37f 100644
--- a/pagure_importer/utils/git.py
+++ b/pagure_importer/utils/git.py
@@ -10,7 +10,10 @@ import pygit2
 import json
 import hashlib
 import werkzeug
-from pagure_importer.utils import is_image
+
+from .utils import is_image, issue_to_json
+from repo import PagureRepo
+
 
 def get_secure_filename(attachment, filename):
     ''' Hashes the file name, same as pagure '''
@@ -46,48 +49,16 @@ def push_repo(newpath, new_repo):
     ori_remote.push([refname])
 
 
-def update_git(obj, newpath, new_repo):
+def update_git(newpath, new_repo):
     """ Update the given issue in its git.
     This method forks the provided repo, add/edit the issue whose file name
     is defined by the uid field of the issue and if there are additions/
     changes commit them and push them back to the original repo.
     """
-    file_path = os.path.join(newpath, obj.uid)
 
     # Get the current index
     index = new_repo.index
 
-    # Are we adding files
-    added = False
-    if not os.path.exists(file_path):
-        added = True
-
-    # If we have attachments
-    attachments = obj.attachment
-    if attachments:
-        if not os.path.exists(os.path.join(newpath, 'files')):
-            os.mkdir(os.path.join(newpath, 'files'))
-
-        for key in attachments.keys():
-            filename = get_secure_filename(attachments[key], key)
-            attach_path = os.path.join(newpath, 'files', filename)
-            # Try decoding Bytes to UTF-8
-            try:
-                with open(attach_path, 'w') as stream:
-                    stream.write(attachments[key].decode())
-            # If it fails write the data as binary
-            except UnicodeDecodeError:
-                with open(attach_path, 'wb') as stream:
-                    stream.write(attachments[key])
-
-            index.add('files/' + filename)
-
-    # Write down what changed
-    with open(file_path, 'w') as stream:
-        stream.write(json.dumps(
-            obj.to_json(), sort_keys=True, indent=4,
-            separators=(',', ': ')))
-
     # Retrieve the list of files that changed
     diff = new_repo.diff()
     files = []
diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index a403836..f33b0db 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -7,8 +7,7 @@ from base64 import b64decode
 from datetime import datetime
 from pagure_importer.utils import (
     get_pagure_namespace, get_close_status, is_image, Importer)
-from pagure_importer.utils.git import (
-    clone_repo, get_secure_filename, push_repo, update_git)
+from pagure_importer.utils.git import get_secure_filename
 from pagure_importer.utils.models import User, Issue, IssueComment
 
 
@@ -80,10 +79,19 @@ class TracImporter(Importer):
                 custom_fields.append(current_field)
         return custom_fields
 
-    def import_issues(self, trac_query='max=0&order=id'):
-        ''' Import issues from trac instance using jsonrpc API '''
+    def import_issues(self, repo_name, repo_folder,
+                      trac_query='max=0&order=id'):
+        ''' Queries the trac instance via its jsonrpc API and convert the
+        tickets into JSON blob to be imported into pagure's ticket git repo.
+
+        :arg repo_name: the name of the repository
+        :arg repo_folder: the folder in which is the repository
+        :kwarg trac_query: the query to call trac with in order to retrieve
+            all the tickets.
+            Defaults to ``max=0&order=id``
+
+        '''
 
-        newpath, new_repo = clone_repo(self.repo_name, self.repo_folder)
         tickets_id = self.request('ticket.query', trac_query)
 
         for ticket_id in tickets_id:
@@ -96,7 +104,6 @@ class TracImporter(Importer):
                 if comments[key].attachment is not None and \
                    any(attachment in comments[key].attachment for attachment in
                        pagure_issue.attachment):
-                    project = get_pagure_namespace(self.repo_folder, self.repo_name)
 
                     for attach_name in comments[key].attachment:
                         filename = get_secure_filename(
@@ -109,11 +116,9 @@ class TracImporter(Importer):
                             comments[key].comment += ('\n[%s](%s)' %
                                                       (attach_name, url))
                 pagure_issue.comments.append(comments[key].to_json())
-            # update the local git repo
-            new_repo = update_git(pagure_issue, newpath, new_repo)
-            click.echo('Updated ' + self.repo_name + ' with issue :' +
+            click.echo('Updated ' + repo_name + ' with issue :' +
                        str(ticket_id) + '/' + str(tickets_id[-1]))
-        push_repo(newpath, new_repo)
+            issue_to_json(obj, repo_folder)
 
     def get_custom_fields_of_ticket(self, trac_ticket):
         ''' Given the trac ticket, it will return all the

From a0a9a53ccb1efd954780c4084ea2254934430a07 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jan 03 2017 17:30:09 +0000
Subject: [PATCH 3/13] Refactor the logic in import_issues for GitHub


Currently when calling import issues the program does several things:
- Clone the git repo or rather lists all cloned repo and asks which one
should be used
- Queries all the tickets in GitHub
- Convert them to JSON blob
- Import them into the git repo selected in the first step
- Push said git repo and delete it from the filesystem

With this commits, the logic is moved into the CLI itself, which now
does select where is the git repo to use, then does the querying of
GitHub and converting to JSON, does the import into git in one step at
the end speeding up the process and finally push and clean as before.

This means that import_issues() can now be used outside of any git repo
and it will just output the JSON blob in the specified directory.

This commit performs the same change as the previous one but for GitHub
instead of trac

---

diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py
index 0b91541..e572647 100644
--- a/pagure_importer/commands/github.py
+++ b/pagure_importer/commands/github.py
@@ -30,13 +30,28 @@ def github(username, password, project):
                 'Choose the import destination repo', default=1)
             repo_name = repos[int(repo_index)-1]
 
+
+            newpath, new_repo = clone_repo(repo_path, repo_folder)
+
             with GithubImporter(username=username,
                                 password=password,
                                 project=project,
                                 repo_name=repo_name,
                                 repo_folder=REPO_PATH) as github_importer:
 
-                github_importer.import_issues()
+                repo = github_importer.github.get_repo(
+                    github_importer.github_project_name)
+                try:
+                    repo_name = repo.name
+                except:
+                    raise GithubRepoNotFound(
+                            'Repo not found, project name wrong')
+                github_importer.import_issues(repo, new_repo)
+
+            # update the local git repo
+            new_repo = update_git(pagure_issue, newpath, new_repo)
+            push_delete_repo(newpath, new_repo)
+
         else:
             click.echo(
                 'No ticket repository found. Use pgimport clone command')
diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py
index c87525d..4193f7f 100644
--- a/pagure_importer/utils/importer_github.py
+++ b/pagure_importer/utils/importer_github.py
@@ -38,23 +38,13 @@ class GithubImporter(Importer):
         if assignee is not None:
             return assignee.to_json()
 
-    def import_issues(self, status='all'):
-        ''' Imports the issues on github for
-        the given project
+    def import_issues(self, repo, repo_folder, status='all'):
+        ''' Imports the issues on github for the given project
         '''
-        repo = self.github.get_repo(self.github_project_name)
-        try:
-            repo_name = repo.name
-        except:
-            raise GithubRepoNotFound(
-                    'Repo not found, project name wrong')
-
-        newpath, new_repo = clone_repo(self.repo_name, self.repo_folder)
+
         repo_issues = repo.get_issues(state=status)
-        issues_length = 0
-        for issue in repo_issues:
-            issues_length += 1
-        for idx, github_issue in enumerate(repo_issues):
+
+        for github_issue in repo_issues:
 
             # title of the issue
             pagure_issue_title = github_issue.title
@@ -153,7 +143,4 @@ class GithubImporter(Importer):
             # add all the comments to the issue object
             pagure_issue.comments = comments
 
-            # update the local git repo
-            new_repo = update_git(pagure_issue, newpath, new_repo)
-            click.echo('Updated issue %s out of %s' % (idx+1, issues_length))
-        push_repo(newpath, new_repo)
+            issue_to_json(pagure_issue, repo_folder)

From 7a7fe3ec7c37f24ce663e6bc6b0fff710aaa86aa Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jan 03 2017 17:30:09 +0000
Subject: [PATCH 4/13] Add a --nopush argument to pagure-importer


This argument allows to do all the steps up to pushing back the imported
ticket to pagure but does not do that push.
This way users can stop after the import has finished and check locally
the state of the data before pushing it to pagure.

---

diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py
index 9164b6c..cb83ac0 100644
--- a/pagure_importer/commands/fedorahosted.py
+++ b/pagure_importer/commands/fedorahosted.py
@@ -18,8 +18,10 @@ import pagure_importer.utils.git as gitutils
               help="FAS password")
 @click.option('--offset', default=0,
               help='Number of issue in pagure before import')
-def fedorahosted(project_url, tags, private, username, password, offset):
-    project_url = project_url.rstrip('/')
+@click.option('--nopush', is_flag=True,
+              help="Do not push the result of pagure-importer back")
+def fedorahosted(
+        project_url, tags, private, username, password, offset, nopush):
     fasclient = FASclient(username, password,
                           'https://admin.fedoraproject.org/accounts')
     project_url += '/login/jsonrpc'
@@ -43,6 +45,7 @@ def fedorahosted(project_url, tags, private, username, password, offset):
             trac_importer.import_issues(project, REPO_PATH)
         # update the local git repo
         new_repo = gitutils.update_git(newpath, new_repo)
-        gitutils.push_delete_repo(newpath, new_repo)
+        if not nopush:
+            gitutils.push_delete_repo(newpath, new_repo)
     else:
         click.echo('No ticket repository found. Use pgimport clone command')
diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py
index e572647..3421bf4 100644
--- a/pagure_importer/commands/github.py
+++ b/pagure_importer/commands/github.py
@@ -15,6 +15,8 @@ from pagure_importer.utils import (
 @click.option('--project',
               prompt='Enter github project name like pypingou/pagure',
               help="Github project like pypingou/pagure")
+@click.option('--nopush', is_flag=True,
+              help="Do not push the result of pagure-importer back")
 def github(username, password, project):
     gen_json = click.confirm(
         "Do you want to generate jsons for project's contributers"
@@ -50,8 +52,9 @@ def github(username, password, project):
 
             # update the local git repo
             new_repo = update_git(pagure_issue, newpath, new_repo)
-            push_delete_repo(newpath, new_repo)
 
+            if not nopush:
+                push_delete_repo(newpath, new_repo)
         else:
             click.echo(
                 'No ticket repository found. Use pgimport clone command')

From 60af086bc31c29e0a5f4e4e641fc704e92c48e47 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jan 03 2017 19:34:05 +0000
Subject: [PATCH 5/13] Fix all the issues found during the review of the refactoring


- Fix variable name that got changed
- Fix showing the progress when importing issues from github
- Fix imports (added missing ones, removed un-needed ones)

---

diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py
index 52fc241..e492963 100644
--- a/pagure_importer/utils/__init__.py
+++ b/pagure_importer/utils/__init__.py
@@ -255,8 +255,8 @@ class Importer:
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         ''' Delete the cloned repo where the commits were going '''
-        if os.path.exists(clone_repo_location):
-            shutil.rmtree(clone_repo_location)
+        if os.path.exists(self.clone_repo_location):
+            shutil.rmtree(self.clone_repo_location)
 
 
 def issue_to_json(issue, folder):
diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py
index 7bee37f..710e01d 100644
--- a/pagure_importer/utils/git.py
+++ b/pagure_importer/utils/git.py
@@ -11,8 +11,7 @@ import json
 import hashlib
 import werkzeug
 
-from .utils import is_image, issue_to_json
-from repo import PagureRepo
+from pagure_importer.utils import is_image, issue_to_json
 
 
 def get_secure_filename(attachment, filename):
diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py
index 4193f7f..b90a3ca 100644
--- a/pagure_importer/utils/importer_github.py
+++ b/pagure_importer/utils/importer_github.py
@@ -2,9 +2,11 @@ import click
 from github import Github
 
 from pagure_importer.utils import (
-    models, gh_get_user_email, get_auth_token, Importer)
+    models, gh_get_user_email, get_auth_token, Importer, issue_to_json
+)
 from pagure_importer.utils.git import (
-    clone_repo, push_repo, update_git)
+    clone_repo, push_repo, update_git
+)
 from pagure_importer.utils.exceptions import (
     GithubRepoNotFound
 )
@@ -106,7 +108,9 @@ class GithubImporter(Importer):
 
             # comments on the issue
             comments = []
-            for comment in github_issue.get_comments():
+            github_comments = github_issue.get_comments()
+            n_comments = len(github_comments)
+            for idx, comment in enumerate(github_comments):
 
                 comment_user = comment.user
                 pagure_issue_comment_body = comment.body
@@ -143,4 +147,5 @@ class GithubImporter(Importer):
             # add all the comments to the issue object
             pagure_issue.comments = comments
 
+            click.echo('Updated issue %s out of %s' % (idx + 1, n_comments))
             issue_to_json(pagure_issue, repo_folder)
diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index f33b0db..4c86f99 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -6,7 +6,8 @@ import requests
 from base64 import b64decode
 from datetime import datetime
 from pagure_importer.utils import (
-    get_pagure_namespace, get_close_status, is_image, Importer)
+    get_pagure_namespace, get_close_status, is_image, Importer,
+    issue_to_json)
 from pagure_importer.utils.git import get_secure_filename
 from pagure_importer.utils.models import User, Issue, IssueComment
 
@@ -108,7 +109,7 @@ class TracImporter(Importer):
                     for attach_name in comments[key].attachment:
                         filename = get_secure_filename(
                             pagure_issue.attachment[attach_name], attach_name)
-                        url = '/%s/issue/raw/files/%s' % (project, filename)
+                        url = '/%s/issue/raw/files/%s' % (repo_name, filename)
                         if is_image(attach_name):
                             comments[key].comment += ('\n[![%s](%s)](%s)' %
                                                       (attach_name, url, url))
@@ -118,7 +119,7 @@ class TracImporter(Importer):
                 pagure_issue.comments.append(comments[key].to_json())
             click.echo('Updated ' + repo_name + ' with issue :' +
                        str(ticket_id) + '/' + str(tickets_id[-1]))
-            issue_to_json(obj, repo_folder)
+            issue_to_json(pagure_issue, repo_folder)
 
     def get_custom_fields_of_ticket(self, trac_ticket):
         ''' Given the trac ticket, it will return all the

From f54ea6ab3b99a5f02338e98140dfaa7974134660 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jan 03 2017 20:24:39 +0000
Subject: [PATCH 6/13] More import clean up and move get_secure_filename into the utils module


---

diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py
index e492963..1829a46 100644
--- a/pagure_importer/utils/__init__.py
+++ b/pagure_importer/utils/__init__.py
@@ -2,10 +2,14 @@ import csv
 import os
 import sys
 import json
-import click
-import pygit2
 import re
 import shutil
+
+import click
+import hashlib
+import pygit2
+import werkzeug
+
 from urllib.parse import urlparse
 from configparser import ConfigParser
 from github import Github
@@ -277,12 +281,12 @@ def issue_to_json(issue, folder):
     # If we have attachments
     attachments = issue.attachment
     if attachments:
-        if not os.path.exists(os.path.join(newpath, 'files')):
-            os.mkdir(os.path.join(newpath, 'files'))
+        if not os.path.exists(os.path.join(folder, 'files')):
+            os.mkdir(os.path.join(folder, 'files'))
 
         for key in attachments.keys():
             filename = get_secure_filename(attachments[key], key)
-            attach_path = os.path.join(newpath, 'files', filename)
+            attach_path = os.path.join(folder, 'files', filename)
             with open(attach_path, 'w') as stream:
                 stream.write(str(attachments[key]))
             files.append('files/' + filename)
@@ -294,3 +298,10 @@ def issue_to_json(issue, folder):
             separators=(',', ': ')))
 
     return files
+
+
+def get_secure_filename(attachment, filename):
+    ''' Hashes the file name, same as pagure '''
+    filename = '%s-%s' % (hashlib.sha256(attachment).hexdigest(),
+                          werkzeug.secure_filename(str(filename)))
+    return filename
diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py
index 710e01d..b49ccad 100644
--- a/pagure_importer/utils/git.py
+++ b/pagure_importer/utils/git.py
@@ -6,19 +6,8 @@
 
 import shutil
 import os
-import pygit2
-import json
-import hashlib
-import werkzeug
-
-from pagure_importer.utils import is_image, issue_to_json
 
-
-def get_secure_filename(attachment, filename):
-    ''' Hashes the file name, same as pagure '''
-    filename = '%s-%s' % (hashlib.sha256(attachment).hexdigest(),
-                          werkzeug.secure_filename(str(filename)))
-    return filename
+import pygit2
 
 
 def clone_repo(repo_name, repo_folder):
diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index 4c86f99..c74d015 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -7,8 +7,7 @@ from base64 import b64decode
 from datetime import datetime
 from pagure_importer.utils import (
     get_pagure_namespace, get_close_status, is_image, Importer,
-    issue_to_json)
-from pagure_importer.utils.git import get_secure_filename
+    issue_to_json, get_secure_filename)
 from pagure_importer.utils.models import User, Issue, IssueComment
 
 

From 17d9c44887a72ce22833f72a70b29a81cb20d457 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jan 04 2017 09:02:41 +0000
Subject: [PATCH 7/13] Fix staging the changed files in update_git and provide the commit message as argument


---

diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py
index cb83ac0..c6f31af 100644
--- a/pagure_importer/commands/fedorahosted.py
+++ b/pagure_importer/commands/fedorahosted.py
@@ -44,7 +44,10 @@ def fedorahosted(
 
             trac_importer.import_issues(project, REPO_PATH)
         # update the local git repo
-        new_repo = gitutils.update_git(newpath, new_repo)
+        new_repo = gitutils.update_git(
+            newpath, new_repo,
+            commit_message='Imported issues from fedorahosted project: %s' %
+                repo_name)
         if not nopush:
             gitutils.push_delete_repo(newpath, new_repo)
     else:
diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py
index 3421bf4..e90ea57 100644
--- a/pagure_importer/commands/github.py
+++ b/pagure_importer/commands/github.py
@@ -51,7 +51,11 @@ def github(username, password, project):
                 github_importer.import_issues(repo, new_repo)
 
             # update the local git repo
-            new_repo = update_git(pagure_issue, newpath, new_repo)
+            new_repo = update_git(
+                newpath,
+                new_repo
+                commit_message='Imported issues from the github project: %s' %
+                    repo_name)
 
             if not nopush:
                 push_delete_repo(newpath, new_repo)
diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py
index b49ccad..3d2a143 100644
--- a/pagure_importer/utils/git.py
+++ b/pagure_importer/utils/git.py
@@ -37,7 +37,7 @@ def push_repo(newpath, new_repo):
     ori_remote.push([refname])
 
 
-def update_git(newpath, new_repo):
+def update_git(newpath, new_repo, commit_message):
     """ Update the given issue in its git.
     This method forks the provided repo, add/edit the issue whose file name
     is defined by the uid field of the issue and if there are additions/
@@ -57,13 +57,11 @@ def update_git(newpath, new_repo):
             files.append(p.delta.new_file.path)
 
     # Add the changes to the index
-    if added:
-        index.add(obj.uid)
     for filename in files:
         index.add(filename)
 
     # If not change, return
-    if not files and not added:
+    if not files:
         shutil.rmtree(newpath)
         return
 
@@ -86,7 +84,7 @@ def update_git(newpath, new_repo):
         'refs/heads/master',
         author,
         author,
-        'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title),
+        commit_message,
         new_repo.index.write_tree(),
         parents)
     index.write()

From e5969d46a4bbb858897d0b2acc9e4becec86a681 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Jan 04 2017 21:17:40 +0000
Subject: [PATCH 8/13] Fixing command to call fedorahosted and github import


Signed-off-by: Clement Verna <cverna@tutanota.com>

---

diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py
index c6f31af..c3d6bd8 100644
--- a/pagure_importer/commands/fedorahosted.py
+++ b/pagure_importer/commands/fedorahosted.py
@@ -43,12 +43,11 @@ def fedorahosted(
                                         private=private) as trac_importer:
 
             trac_importer.import_issues(project, REPO_PATH)
-        # update the local git repo
-        new_repo = gitutils.update_git(
-            newpath, new_repo,
-            commit_message='Imported issues from fedorahosted project: %s' %
-                repo_name)
-        if not nopush:
-            gitutils.push_delete_repo(newpath, new_repo)
+            # update the local git repo
+            new_repo = gitutils.update_git(
+                new_repo,
+                commit_message='Imported issues from fedorahosted project: %s' % repo_name)
+            if not nopush:
+                gitutils.push_repo(new_repo)
     else:
         click.echo('No ticket repository found. Use pgimport clone command')
diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py
index e90ea57..5b2a4df 100644
--- a/pagure_importer/commands/github.py
+++ b/pagure_importer/commands/github.py
@@ -6,6 +6,10 @@ from pagure_importer.utils import (
     gh_get_contributors, gh_get_issue_users, gh_assemble_users,
 )
 
+import pagure_importer.utils.git as gitutils
+from pagure_importer.utils.exceptions import (
+    GithubRepoNotFound
+)
 
 @app.command()
 @click.option('--username', prompt='Enter your Github Username',
@@ -17,7 +21,7 @@ from pagure_importer.utils import (
               help="Github project like pypingou/pagure")
 @click.option('--nopush', is_flag=True,
               help="Do not push the result of pagure-importer back")
-def github(username, password, project):
+def github(username, password, project, nopush):
     gen_json = click.confirm(
         "Do you want to generate jsons for project's contributers"
         " and issue commentors?")
@@ -32,8 +36,7 @@ def github(username, password, project):
                 'Choose the import destination repo', default=1)
             repo_name = repos[int(repo_index)-1]
 
-
-            newpath, new_repo = clone_repo(repo_path, repo_folder)
+            newpath, new_repo = gitutils.clone_repo(repo_name, REPO_PATH)
 
             with GithubImporter(username=username,
                                 password=password,
@@ -50,15 +53,13 @@ def github(username, password, project):
                             'Repo not found, project name wrong')
                 github_importer.import_issues(repo, new_repo)
 
-            # update the local git repo
-            new_repo = update_git(
-                newpath,
-                new_repo
-                commit_message='Imported issues from the github project: %s' %
-                    repo_name)
+                # update the local git repo
+                new_repo = gitutils.update_git(
+                    new_repo,
+                    commit_message='Imported issues from the github project: %s' % repo_name)
 
-            if not nopush:
-                push_delete_repo(newpath, new_repo)
+                if not nopush:
+                    gitutils.push_repo(new_repo)
         else:
             click.echo(
                 'No ticket repository found. Use pgimport clone command')

From 264ce04b1ee7c7b76a17c451f88431270d5b00e1 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Jan 04 2017 21:18:25 +0000
Subject: [PATCH 9/13] Cleanup gitutils and remove not needed code


Signed-off-by: Clement Verna <cverna@tutanota.com>

---

diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py
index 3d2a143..aed4aa2 100644
--- a/pagure_importer/utils/git.py
+++ b/pagure_importer/utils/git.py
@@ -19,14 +19,13 @@ def clone_repo(repo_name, repo_folder):
 
     # Get the fork
     repopath = os.path.join(repo_folder, repo_name)
-
     # Clone the repo into a temp folder
     newpath = os.path.join(repo_folder, 'clone-' + repo_name)
     new_repo = pygit2.clone_repository(repopath, newpath)
     return (newpath, new_repo)
 
 
-def push_repo(newpath, new_repo):
+def push_repo(new_repo):
     ''' Push the changes to the originally cloned repo from pagure '''
 
     # Push to origin
@@ -37,7 +36,7 @@ def push_repo(newpath, new_repo):
     ori_remote.push([refname])
 
 
-def update_git(newpath, new_repo, commit_message):
+def update_git(new_repo, commit_message):
     """ Update the given issue in its git.
     This method forks the provided repo, add/edit the issue whose file name
     is defined by the uid field of the issue and if there are additions/
@@ -46,24 +45,7 @@ def update_git(newpath, new_repo, commit_message):
 
     # Get the current index
     index = new_repo.index
-
-    # Retrieve the list of files that changed
-    diff = new_repo.diff()
-    files = []
-    for p in diff:
-        if hasattr(p, 'new_file_path'):
-            files.append(p.new_file_path)
-        elif hasattr(p, 'delta'):
-            files.append(p.delta.new_file.path)
-
-    # Add the changes to the index
-    for filename in files:
-        index.add(filename)
-
-    # If not change, return
-    if not files:
-        shutil.rmtree(newpath)
-        return
+    index.add_all()
 
     # See if there is a parent to this commit
     parent = None

From 327f4064b95f51fe72dc92915d53da9c998ed047 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Jan 05 2017 20:25:13 +0000
Subject: [PATCH 10/13] Do no delete the cloned repo if nopush flag is true


Signed-off-by: Clement Verna <cverna@tutanota.com>

---

diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py
index c3d6bd8..49cbcda 100644
--- a/pagure_importer/commands/fedorahosted.py
+++ b/pagure_importer/commands/fedorahosted.py
@@ -40,7 +40,8 @@ def fedorahosted(
                                         repo_folder=REPO_PATH,
                                         fasclient=fasclient,
                                         tags=tags,
-                                        private=private) as trac_importer:
+                                        private=private,
+                                        nopush=nopush) as trac_importer:
 
             trac_importer.import_issues(project, REPO_PATH)
             # update the local git repo
diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py
index 5b2a4df..83e0a52 100644
--- a/pagure_importer/commands/github.py
+++ b/pagure_importer/commands/github.py
@@ -42,7 +42,8 @@ def github(username, password, project, nopush):
                                 password=password,
                                 project=project,
                                 repo_name=repo_name,
-                                repo_folder=REPO_PATH) as github_importer:
+                                repo_folder=REPO_PATH,
+                                nopush=nopush) as github_importer:
 
                 repo = github_importer.github.get_repo(
                     github_importer.github_project_name)
diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py
index 1829a46..a81887e 100644
--- a/pagure_importer/utils/__init__.py
+++ b/pagure_importer/utils/__init__.py
@@ -246,13 +246,14 @@ def is_image(filename):
 class Importer:
     ''' Common Class for Github and Feodrahosted importer'''
 
-    def __init__(self, username, password, repo_name, repo_folder):
+    def __init__(self, username, password, repo_name, repo_folder, nopush):
         self.username = username
         self.password = password
         self.repo_name = repo_name
         self.repo_folder = repo_folder
         self.clone_repo_location = os.path.join(
             repo_folder, 'clone-' + repo_name)
+        self.nopush = nopush
 
     def __enter__(self):
         return self
@@ -260,7 +261,8 @@ class Importer:
     def __exit__(self, exc_type, exc_val, exc_tb):
         ''' Delete the cloned repo where the commits were going '''
         if os.path.exists(self.clone_repo_location):
-            shutil.rmtree(self.clone_repo_location)
+            if not self.nopush:
+                shutil.rmtree(self.clone_repo_location)
 
 
 def issue_to_json(issue, folder):

From d49e98b547d83c3992dce2cbfed4a71b60b549a0 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Jan 05 2017 20:26:05 +0000
Subject: [PATCH 11/13] Fix cloned repo location


Signed-off-by: Clement Verna <cverna@tutanota.com>

---

diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py
index b90a3ca..e991dd5 100644
--- a/pagure_importer/utils/importer_github.py
+++ b/pagure_importer/utils/importer_github.py
@@ -148,4 +148,4 @@ class GithubImporter(Importer):
             pagure_issue.comments = comments
 
             click.echo('Updated issue %s out of %s' % (idx + 1, n_comments))
-            issue_to_json(pagure_issue, repo_folder)
+            issue_to_json(pagure_issue, self.clone_repo_location)
diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index c74d015..5024e40 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -118,7 +118,7 @@ class TracImporter(Importer):
                 pagure_issue.comments.append(comments[key].to_json())
             click.echo('Updated ' + repo_name + ' with issue :' +
                        str(ticket_id) + '/' + str(tickets_id[-1]))
-            issue_to_json(pagure_issue, repo_folder)
+            issue_to_json(pagure_issue, self.clone_repo_location)
 
     def get_custom_fields_of_ticket(self, trac_ticket):
         ''' Given the trac ticket, it will return all the

From 2ab2a65de1283576ee0df5fbc78252ee8033c5a5 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Jan 05 2017 20:46:53 +0000
Subject: [PATCH 12/13] Cleaning importers code and passing nopush  arg


Signed-off-by: Clement Verna <cverna@tutanota.com>

---

diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py
index e991dd5..479e844 100644
--- a/pagure_importer/utils/importer_github.py
+++ b/pagure_importer/utils/importer_github.py
@@ -4,21 +4,15 @@ from github import Github
 from pagure_importer.utils import (
     models, gh_get_user_email, get_auth_token, Importer, issue_to_json
 )
-from pagure_importer.utils.git import (
-    clone_repo, push_repo, update_git
-)
-from pagure_importer.utils.exceptions import (
-    GithubRepoNotFound
-)
 
 
 class GithubImporter(Importer):
     ''' Imports from Github using PyGithub and libpagure '''
 
-    def __init__(self, username, password, project, repo_name, repo_folder):
+    def __init__(self, username, password, project, repo_name, repo_folder, nopush):
         ''' Instantiate GithubImporter object '''
 
-        Importer.__init__(self, username, password, repo_name, repo_folder)
+        Importer.__init__(self, username, password, repo_name, repo_folder, nopush)
         self.github_project_name = project
         self.github = Github(username, password)
 
@@ -40,7 +34,7 @@ class GithubImporter(Importer):
         if assignee is not None:
             return assignee.to_json()
 
-    def import_issues(self, repo, repo_folder, status='all'):
+    def import_issues(self, repo, status='all'):
         ''' Imports the issues on github for the given project
         '''
 
diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index 5024e40..75dd30b 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -6,8 +6,8 @@ import requests
 from base64 import b64decode
 from datetime import datetime
 from pagure_importer.utils import (
-    get_pagure_namespace, get_close_status, is_image, Importer,
-    issue_to_json, get_secure_filename)
+    get_close_status, is_image, Importer, issue_to_json,
+    get_secure_filename)
 from pagure_importer.utils.models import User, Issue, IssueComment
 
 
@@ -24,9 +24,9 @@ class TracImporter(Importer):
     ''' Pagure importer for trac instance '''
 
     def __init__(self, project_url, username, password, offset, repo_name,
-                 repo_folder, fasclient=None, tags=False, private=False):
+                 repo_folder, nopush, fasclient=None, tags=False, private=False):
         ''' Instantiate a TracImporter object '''
-        Importer.__init__(self, username, password, repo_name, repo_folder)
+        Importer.__init__(self, username, password, repo_name, repo_folder, nopush)
         self.url = project_url
         self.fas = fasclient
         self.tags = tags
@@ -60,7 +60,6 @@ class TracImporter(Importer):
 
         return resp['result']
 
-
     def get_custom_fields(self):
         ''' Queries the fedorahosted api to get all ticket fields
         and filters all the custom fields, returns

From 9e9495af296715876025af0f765b4c584b502485 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Jan 07 2017 08:43:08 +0000
Subject: [PATCH 13/13] Removed unused variable and fix issue count logic for github import


Signed-off-by: Clement Verna <cverna@tutanota.com>

---

diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py
index 49cbcda..b3b1986 100644
--- a/pagure_importer/commands/fedorahosted.py
+++ b/pagure_importer/commands/fedorahosted.py
@@ -43,7 +43,7 @@ def fedorahosted(
                                         private=private,
                                         nopush=nopush) as trac_importer:
 
-            trac_importer.import_issues(project, REPO_PATH)
+            trac_importer.import_issues(project)
             # update the local git repo
             new_repo = gitutils.update_git(
                 new_repo,
diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py
index 83e0a52..070f130 100644
--- a/pagure_importer/commands/github.py
+++ b/pagure_importer/commands/github.py
@@ -52,7 +52,7 @@ def github(username, password, project, nopush):
                 except:
                     raise GithubRepoNotFound(
                             'Repo not found, project name wrong')
-                github_importer.import_issues(repo, new_repo)
+                github_importer.import_issues(repo)
 
                 # update the local git repo
                 new_repo = gitutils.update_git(
diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py
index 479e844..adcebc2 100644
--- a/pagure_importer/utils/importer_github.py
+++ b/pagure_importer/utils/importer_github.py
@@ -39,8 +39,9 @@ class GithubImporter(Importer):
         '''
 
         repo_issues = repo.get_issues(state=status)
+        issues_length = sum(1 for issue in repo_issues)
 
-        for github_issue in repo_issues:
+        for idx, github_issue in enumerate(repo_issues):
 
             # title of the issue
             pagure_issue_title = github_issue.title
@@ -102,9 +103,7 @@ class GithubImporter(Importer):
 
             # comments on the issue
             comments = []
-            github_comments = github_issue.get_comments()
-            n_comments = len(github_comments)
-            for idx, comment in enumerate(github_comments):
+            for comment in github_issue.get_comments():
 
                 comment_user = comment.user
                 pagure_issue_comment_body = comment.body
@@ -141,5 +140,5 @@ class GithubImporter(Importer):
             # add all the comments to the issue object
             pagure_issue.comments = comments
 
-            click.echo('Updated issue %s out of %s' % (idx + 1, n_comments))
+            click.echo('Updated %s with issue : %s/%s' % (self.repo_name, idx + 1, issues_length))
             issue_to_json(pagure_issue, self.clone_repo_location)
diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index 75dd30b..eb1d20c 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -78,8 +78,7 @@ class TracImporter(Importer):
                 custom_fields.append(current_field)
         return custom_fields
 
-    def import_issues(self, repo_name, repo_folder,
-                      trac_query='max=0&order=id'):
+    def import_issues(self, repo_name, trac_query='max=0&order=id'):
         ''' Queries the trac instance via its jsonrpc API and convert the
         tickets into JSON blob to be imported into pagure's ticket git repo.