From b20b6fde6899fe1635736ddc3cbbf3d39ae19a2c Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jul 18 2016 07:13:42 +0000 Subject: [PATCH 1/12] Added config file for FAS Credentials and moved scripts to a folder --- diff --git a/category_scraper.py b/category_scraper.py deleted file mode 100644 index dbfad72..0000000 --- a/category_scraper.py +++ /dev/null @@ -1,11 +0,0 @@ -from bs4 import BeautifulSoup -from urllib import urlopen - -page = urlopen('https://fedora-fedmsg.readthedocs.io/en/latest/topics.html') -soup = BeautifulSoup(page.read().decode('ascii', 'ignore'), 'html.parser') -categorylist = list() -for h3 in soup.findAll('h3'): - if len(h3.text.split('.')) > 2 and h3.text.split('.') not in categorylist: - categorylist.append(h3.text.split('.')) - -print categorylist diff --git a/fas_credentials.cfg b/fas_credentials.cfg new file mode 100644 index 0000000..2415ba9 --- /dev/null +++ b/fas_credentials.cfg @@ -0,0 +1,11 @@ +''' +Please note that this is required only for scraping the data from groups using +python-fedora API. You do not have to fill it up if you are not going to +pull data of users in a particular FAS group. +''' + +[fas] +# Enter your FAS credentials here + +username = 'your_fas_username_here' +password = 'your_fas_password_here' diff --git a/scripts/category_scraper.py b/scripts/category_scraper.py new file mode 100644 index 0000000..dbfad72 --- /dev/null +++ b/scripts/category_scraper.py @@ -0,0 +1,11 @@ +from bs4 import BeautifulSoup +from urllib import urlopen + +page = urlopen('https://fedora-fedmsg.readthedocs.io/en/latest/topics.html') +soup = BeautifulSoup(page.read().decode('ascii', 'ignore'), 'html.parser') +categorylist = list() +for h3 in soup.findAll('h3'): + if len(h3.text.split('.')) > 2 and h3.text.split('.') not in categorylist: + categorylist.append(h3.text.split('.')) + +print categorylist diff --git a/scripts/us_pycon_autostats.py b/scripts/us_pycon_autostats.py new file mode 100644 index 0000000..8fc1ef8 --- /dev/null +++ b/scripts/us_pycon_autostats.py @@ -0,0 +1,15 @@ +import os + + +def main(): + + f = open('users.txt') + lines = [line.rstrip('\n') for line in open('users.txt')] + print lines + + for user in lines: + os.system( + 'python main.py -u ' + + user + + ' -s 05/28/2016 -e 06/05/2016 -m csv') +main() diff --git a/scripts/weekly_intern_stats.py b/scripts/weekly_intern_stats.py new file mode 100644 index 0000000..fa7d8b1 --- /dev/null +++ b/scripts/weekly_intern_stats.py @@ -0,0 +1,27 @@ +import os + + +def main(): + fp = open('interns.txt') + interns = [user.strip('\n') for user in open('interns.txt')] + print interns + + for intern in interns: + if not os.path.exists(intern): + os.makedirs(intern) + # CSV MAIN FILE + os.system('python main.py -u %s -s 05/23/2016 -e 06/21/2016 -m csv -o stats_main' %( + intern)) + # Category-wise and Pagure + os.system('python main.py -u %s -s 05/23/2016 -e 06/21/2016 -m svg -o %s/%s -c pagure' %( + intern, intern, intern)) + # Markdown Report + os.system('python main.py -u %s -s 05/23/2016 -e 06/23/2016 -m markdown -o %s/README.md' %( + intern, intern)) + # User Specific + if intern == 'dhanvi': + os.system('python main.py -u %s -s 05/23/2016 -e 06/21/2016 -m svg -o %s/%s -c copr' %( + intern, intern, intern)) + +if __name__ == '__main__': + main() diff --git a/us_pycon_autostats.py b/us_pycon_autostats.py deleted file mode 100644 index 8fc1ef8..0000000 --- a/us_pycon_autostats.py +++ /dev/null @@ -1,15 +0,0 @@ -import os - - -def main(): - - f = open('users.txt') - lines = [line.rstrip('\n') for line in open('users.txt')] - print lines - - for user in lines: - os.system( - 'python main.py -u ' + - user + - ' -s 05/28/2016 -e 06/05/2016 -m csv') -main() diff --git a/weekly_intern_stats.py b/weekly_intern_stats.py deleted file mode 100644 index 8807e47..0000000 --- a/weekly_intern_stats.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - - -def main(): - fp = open('interns.txt') - interns = [user.strip('\n') for user in open('interns.txt')] - print interns - - for intern in interns: - if not os.path.exists(intern): - os.makedirs(intern) - # CSV MAIN FILE - os.system('python main.py -u %s -s 05/23/2016 -e 06/21/2016 -m csv -o stats_main' %( - intern)) - # Category-wise and Pagure - os.system('python main.py -u %s -s 05/23/2016 -e 06/21/2016 -m svg -o %s/%s -c pagure' %( - intern, intern, intern)) - # Markdown Report - os.system('python main.py -u %s -s 05/23/2016 -e 06/21/2016 -m markdown -o %s/README.md' %( - intern, intern)) - # User Specific - if intern == 'dhanvi': - os.system('python main.py -u %s -s 05/23/2016 -e 06/21/2016 -m svg -o %s/%s -c copr' %( - intern, intern, intern)) - -if __name__ == '__main__': - main() From 6f2fe71797feaeeff9b5a38236f18ac2ddceb710 Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jul 18 2016 07:24:55 +0000 Subject: [PATCH 2/12] Removed fas_credentials.cfg from version control --- diff --git a/.gitignore b/.gitignore index e20eb79..5ce3e24 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ *.csv *.md *.gource +*.cfg diff --git a/fas_credentials.cfg b/fas_credentials.cfg deleted file mode 100644 index 2415ba9..0000000 --- a/fas_credentials.cfg +++ /dev/null @@ -1,11 +0,0 @@ -''' -Please note that this is required only for scraping the data from groups using -python-fedora API. You do not have to fill it up if you are not going to -pull data of users in a particular FAS group. -''' - -[fas] -# Enter your FAS credentials here - -username = 'your_fas_username_here' -password = 'your_fas_password_here' From e2cc3a6ca5aaad1f299b6cba47d1a08e3e1c4840 Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jul 18 2016 11:14:04 +0000 Subject: [PATCH 3/12] Added group parsing --- diff --git a/main.py b/main.py index 2d4f055..c867965 100644 --- a/main.py +++ b/main.py @@ -36,17 +36,18 @@ def assign_values(args): def add_arguments(parser): - parser.add_argument('--user', '-u', help='FAS username') - parser.add_argument('--weeks', '-w', help='Time in weeks', default=1) - parser.add_argument('--mode', '-m', help="Type of Output", default='text') - parser.add_argument('--output', '-o', help="Output name", default='stats') parser.add_argument('--category', '-c', help="Sub Category", default='') - parser.add_argument('--start', '-s', help="Start Date", default='') parser.add_argument('--end', '-e', help="End Date", default='') - parser.add_argument('--log', '-l', help="Enable full log reporting", - action='store_true') + parser.add_argument('--group', '-g', help="FAS Group", default='') parser.add_argument('--interactive', '-i', help="Enable interactive mode", - action='store_true') + action='store_true') + parser.add_argument('--log', '-l', help="Enable full log reporting", + action='store_true') + parser.add_argument('--mode', '-m', help="Type of Output", default='text') + parser.add_argument('--output', '-o', help="Output name", default='stats') + parser.add_argument('--start', '-s', help="Start Date", default='') + parser.add_argument('--user', '-u', help='FAS username') + parser.add_argument('--weeks', '-w', help='Time in weeks', default=1) def main(): diff --git a/parseGroup.py b/parseGroup.py new file mode 100644 index 0000000..1f08876 --- /dev/null +++ b/parseGroup.py @@ -0,0 +1,39 @@ +import ConfigParser +import os.path +from fedora.client import AccountSystem, AuthError + +class GroupParser: + + def __init__(self): + self.username = None + self.password = None + self.config = ConfigParser.RawConfigParser() + try: + self.config.read('fas_credentials.cfg') + self.username = self.config.get('fas', 'username').strip('\'') + self.password = self.config.get('fas', 'password').strip('\'') + self.check_config() + except: + print("[*] Invalid / Missing Configuration file.") + def check_config(self): + if self.username.strip('\'') == 'FAS_USERNAME_HERE': + print("[*] Please enter FAS credentials in fas_credentials.cfg") + return False + else: + return True + + def group_users(self, group_name): + userlist = list() + group_json = {} + account = AccountSystem(username=self.username, + password=self.password) + try: + group_json = account.group_members(group_name) + except AuthError: + print("[*] Invalid Username / Password") + for user_desc in group_json: + userlist.append(user_desc.values()[0]) + return userlist + +obj = GroupParser() +print obj.group_users('commops') From 7a93e8f5c3ae6491cc39f5fe12332f22440dc577 Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jul 30 2016 11:33:25 +0000 Subject: [PATCH 4/12] Added group parsing, fixed minor bugs and improved file structure --- diff --git a/main.py b/main.py index c867965..682b6e6 100644 --- a/main.py +++ b/main.py @@ -9,6 +9,7 @@ import calendar import fedmsg.meta import stats import output +from parseGroup import GroupParser def interactive_input(args): @@ -30,9 +31,10 @@ def assign_values(args): stats.start = args.start stats.end = args.end stats.weeks = int(args.weeks) + stats.group = args.group stats.log = args.log output.mode = args.mode.lower() - output.filename = args.output.lower() + output.filename = args.user.lower() def add_arguments(parser): @@ -50,41 +52,22 @@ def add_arguments(parser): parser.add_argument('--weeks', '-w', help='Time in weeks', default=1) -def main(): - # fedmsg config - config = fedmsg.config.load_config() - fedmsg.meta.make_processors(**config) - - # Initializing to None to prevent errors while generating multiple reports - stats.unicode_json = {} - i_count = 0 - - # Argument Parser initialization - parser = argparse.ArgumentParser(description='Fedora Statistics Gatherer') - add_arguments(parser) - args = parser.parse_args() - - # Check if the argument type is interactive - if args.interactive: - interactive_input(args) - - # Check if the user argument exists - elif args.user is None: - print(colored("[!] ", 'red') + "Username is required. Use -h for help") - return 1 +def generator(args, mode, user): + if mode=='group': + args.user = user + stats.values['user'] = user # Else, use the argparse values. No arguments is handled by argparse. - else: - assign_values(args) + assign_values(args) # For png and SVG, we need a drawable object to be called if output.mode in ['png', 'svg', 'csv'] or not stats.log and output.mode == 'text': - # Draw object for the above mentioned categories. + # Draw object for the above mentioned categories. draw_obj = stats.return_categories() # To handle user with no activity; TO-DO -> Make this a function if len(draw_obj) == 0: - print (colored("[!] ", 'red') + 'No activity found for user' + - args.user) + print (colored("[!] ", 'red') + 'No activity found for user ' + + args.user) return 1 # Generate the output graph objects required for calling generate_graph @@ -129,6 +112,46 @@ def main(): str(args.user)) return 1 output.generate_graph(draw_obj, args.user) + stats.unicode_json = {} + + +def main(): + # fedmsg config + config = fedmsg.config.load_config() + fedmsg.meta.make_processors(**config) + + # Initializing to None to prevent errors while generating multiple reports + stats.unicode_json = {} + i_count = 0 + group_userlist = list() + + # Argument Parser initialization + parser = argparse.ArgumentParser(description='Fedora Statistics Gatherer') + add_arguments(parser) + args = parser.parse_args() + + if args.group: + print("Gathering group statistics ..") + group = GroupParser() + group_userlist = list(group.group_users(args.group)) + print(group_userlist) + + # Check if the argument type is interactive + if args.interactive: + interactive_input(args) + + elif args.group: + for user in group_userlist: + generator(args, 'group', user) + # Check if the user argument exists + elif args.user is None : + if not args.group: + print(colored("[!] ", 'red') + "Username is required. Use -h for help") + return 1 + else: + generator(args, 'user', args.user.lower()) + + if __name__ == '__main__': diff --git a/output.py b/output.py index e0157cf..239c506 100644 --- a/output.py +++ b/output.py @@ -13,6 +13,7 @@ import os # Default global variables subcategory_json = None category_json = None +path = '' filename = stats.values['user'] csv_init = text_init = False mode = 'text' @@ -22,19 +23,18 @@ cat = None # Gets a drawable object argument and renders an SVG Image of it. def draw_svg(graph_obj): if cat is None: - fname = filename + '_main' + '.svg' + fname = path + stats.values['user'] + '/' + filename + '_main' + '.svg' else: - fname = filename + "_" + cat + '.svg' + fname = path + stats.values['user'] + '/' + filename + "_" + cat + '.svg' graph_obj.render_to_file(fname) - os.system("firefox " + fname) # Gets a drawable object argument and renders a PNG image of it. def draw_png(graph_obj): if cat is None: - fname = filename + '_main' + '.png' + fname = path + stats.values['user'] + '/' + filename + '_main' + '.png' else: - fname = filename + "_" + cat + '.png' + fname = path + stats.values['user'] + '/' + filename + "_" + cat + '.png' graph_obj.render_to_png(fname) @@ -60,7 +60,7 @@ def draw_bar(output_json, title): # Generates CSV report for the user from the dictionary passed def save_csv(output_json): global csv_init, cat - fname = filename + '_main.csv' + fname = path + stats.values['user'] + '/' + filename + '_main.csv' fout = open(fname, 'a') csvw = csv.writer(fout) @@ -101,7 +101,7 @@ def show_gource(unicode_json): colors = colors * n_wraps color_lookup = dict(zip(procs, colors)) - fname = filename + '_main.gource' + fname = path + stats.values['user'] + '/' + filename + '_main.gource' fout = open(fname, 'w') for activity in unicode_json['raw_messages']: try: @@ -121,7 +121,7 @@ def show_gource(unicode_json): # Saves category-wise text report of a user. def save_text_log(unicode_json): - fname = filename + '_main.txt' + fname = path + stats.values['user'] + '/' + filename + '_main.txt' fout = open(fname, 'w') # Category-wise Log fout.write("\n\n*** Category-wise activities ***\n\n") @@ -149,7 +149,8 @@ def save_text_log(unicode_json): def save_text_metrics(output_json): global text_init - fname = filename + '_main.txt' + fname = path + stats.values['user'] + '/' + filename + '_main.txt' + print(fname) fout = open(fname, 'a') # Write the dates into CSV if not text_init and stats.end and stats.start: @@ -182,7 +183,7 @@ def save_text_metrics(output_json): # Saves the markdown version of the text log def save_markdown(unicode_json): - fname = filename + '_main.md' + fname = path + stats.values['user'] + '/' + filename + '_main.md' fout = open(fname, 'w') # Category-wise Log, markdown ready fout.write("\n\n### Category-wise activities\n\n") @@ -211,7 +212,7 @@ def save_markdown(unicode_json): # Saves the JSON as a file. def save_json(unicode_json): - fname = filename + '_main.json' + fname = path + stats.values['user'] + '/' + filename + '_main.json' try: with open(fname, 'w') as outfile: json.dump(unicode_json, outfile) @@ -221,6 +222,17 @@ def save_json(unicode_json): # Identifies categories & generates drawable objects for the above functions. def generate_graph(output_json, title, category=None, gtype=None): + global path + if stats.group: + path = stats.group + '/' + if not os.path.exists(stats.group): + os.makedirs(stats.group) + if not os.path.exists(path + stats.values['user']): + os.makedirs(path + stats.values['user']) + else: + if not os.path.exists(stats.values['user']): + os.makedirs(stats.values['user']) + global cat cat = category graph_obj = None diff --git a/parseGroup.py b/parseGroup.py index 1f08876..356b8cb 100644 --- a/parseGroup.py +++ b/parseGroup.py @@ -23,17 +23,13 @@ class GroupParser: return True def group_users(self, group_name): - userlist = list() - group_json = {} + group_json = dict() account = AccountSystem(username=self.username, password=self.password) try: group_json = account.group_members(group_name) except AuthError: print("[*] Invalid Username / Password") - for user_desc in group_json: - userlist.append(user_desc.values()[0]) + return 1 + userlist = [user_desc.values()[0] for user_desc in group_json] return userlist - -obj = GroupParser() -print obj.group_users('commops') diff --git a/stats.py b/stats.py index 29b1db8..a7264af 100644 --- a/stats.py +++ b/stats.py @@ -17,6 +17,7 @@ values['page'] = 1 values['size'] = 'small' category = '' start = '' +group = '' end = '' logs = False weeks = 0 @@ -41,8 +42,8 @@ def return_json(): total_pages = 1 # Only pull the values from datagrepper if it's the first run - if len(unicode_json) == 0: - print('[*] Grabbing datagrepper values..') + if len(unicode_json) == 0 or unicode_json['arguments']['users'][0]!=values['user']: + print('[*] Grabbing datagrepper values for user ' + values['user'] + '..') # If the user is set as all, we filter it using the provided category, # if any @@ -65,7 +66,7 @@ def return_json(): print ("Total pages found : " + str(total_pages)) total = total_pages # If multiple pages exist, get them all. - while total_pages > 1: + while total_pages > 0: print(" [*] Loading Page " + str(values['page']) + "/" + str(total)) values['page'] += 1 response = requests.get(baseurl, params=values) @@ -74,6 +75,7 @@ def return_json(): for activity in paginated_json['raw_messages']: unicode_json['raw_messages'].append(activity) total_pages -= 1 + values['page'] = 1 return unicode_json # Analyzes the JSON and return categories present as a list. From 49b947b3360df7f26021811e0e52b1ad9aeaa43b Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jul 30 2016 13:19:24 +0000 Subject: [PATCH 5/12] Added FAS credentials files --- diff --git a/fas_credentials.cfg b/fas_credentials.cfg new file mode 100644 index 0000000..df4bba3 --- /dev/null +++ b/fas_credentials.cfg @@ -0,0 +1,9 @@ +############################################################################### +# Please note that this is required only for scraping the data from groups using +# python-fedora API. You do not have to fill it up if you are not going to +# pull data of users of a particular FAS group. +############################################################################### + +[fas] +username = 'FAS_USERNAME_HERE' +password = 'FAS_PASSWORD_HERE' From 63b36774ea2f81ce43d4cf0487a85f6983009c3e Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jul 30 2016 13:21:25 +0000 Subject: [PATCH 6/12] Removed group debug values --- diff --git a/main.py b/main.py index 682b6e6..4030f1f 100644 --- a/main.py +++ b/main.py @@ -134,7 +134,6 @@ def main(): print("Gathering group statistics ..") group = GroupParser() group_userlist = list(group.group_users(args.group)) - print(group_userlist) # Check if the argument type is interactive if args.interactive: From a7b238c93df86a747c67a90ef7e1fc18f250a97b Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Aug 01 2016 08:40:03 +0000 Subject: [PATCH 7/12] Fixed typos in README. Thanks KK! --- diff --git a/README.md b/README.md index 466c9c3..748dcbd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ## fedstats-gsoc -A simple CLI to tool to gather statistics from [datagrepper](https://apps.fedoraproject.org/datagrepper/) +A simple CLI tool to gather statistics from [datagrepper](https://apps.fedoraproject.org/datagrepper/) ### Description This tool helps pull statistics of any Fedora user with an active [FAS Account](https://fedoraproject.org/wiki/Account_System) account. @@ -9,7 +9,7 @@ This tool helps pull statistics of any Fedora user with an active [FAS Account]( ### Usage This tool uses`argparse` to parse arguments. This can be used in two ways, one-liner / interactive method. -The interactive mode can be enabled by using the `--interactive` or `-i` flag. This mode of usage does not require any additional argument. Please not that the arguments passed (if any) will be invalid. +The interactive mode can be enabled by using the `--interactive` or `-i` flag. This mode of usage does not require any additional argument. Please note that the arguments passed (if any) will be invalid. One-liner uses the classic argument parsing method to generate output. This is useful for automating the report generation process. The only mandatory argument is `--user / -u` which takes the FAS username as input. @@ -60,7 +60,7 @@ be two files with similar file names, hence preventing unexpected over-writes. The naming convention is as follows : -* All the main report files (i.e : Category Overiew), text files are named as `_main.` +* All the main report files (i.e : Category Overview), text files are named as `_main.` * The sub-category report (i.e the Category bar-chart) is named as `_.` From a4c6f29d76924e7066301051dd52d15f6069e2a7 Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Aug 14 2016 15:19:09 +0000 Subject: [PATCH 8/12] Code cleanup and pep-8 --- diff --git a/main.py b/main.py index 4030f1f..507dcf9 100644 --- a/main.py +++ b/main.py @@ -42,9 +42,9 @@ def add_arguments(parser): parser.add_argument('--end', '-e', help="End Date", default='') parser.add_argument('--group', '-g', help="FAS Group", default='') parser.add_argument('--interactive', '-i', help="Enable interactive mode", - action='store_true') + action='store_true') parser.add_argument('--log', '-l', help="Enable full log reporting", - action='store_true') + action='store_true') parser.add_argument('--mode', '-m', help="Type of Output", default='text') parser.add_argument('--output', '-o', help="Output name", default='stats') parser.add_argument('--start', '-s', help="Start Date", default='') @@ -53,7 +53,7 @@ def add_arguments(parser): def generator(args, mode, user): - if mode=='group': + if mode == 'group': args.user = user stats.values['user'] = user @@ -61,13 +61,13 @@ def generator(args, mode, user): assign_values(args) # For png and SVG, we need a drawable object to be called if output.mode in ['png', 'svg', 'csv'] or not stats.log and output.mode == 'text': - # Draw object for the above mentioned categories. + # Draw object for the above mentioned categories. draw_obj = stats.return_categories() # To handle user with no activity; TO-DO -> Make this a function if len(draw_obj) == 0: print (colored("[!] ", 'red') + 'No activity found for user ' + - args.user) + args.user) return 1 # Generate the output graph objects required for calling generate_graph @@ -143,7 +143,7 @@ def main(): for user in group_userlist: generator(args, 'group', user) # Check if the user argument exists - elif args.user is None : + elif args.user is None: if not args.group: print(colored("[!] ", 'red') + "Username is required. Use -h for help") return 1 @@ -151,7 +151,5 @@ def main(): generator(args, 'user', args.user.lower()) - - if __name__ == '__main__': main() diff --git a/output.py b/output.py index 239c506..8a5154e 100644 --- a/output.py +++ b/output.py @@ -23,18 +23,22 @@ cat = None # Gets a drawable object argument and renders an SVG Image of it. def draw_svg(graph_obj): if cat is None: - fname = path + stats.values['user'] + '/' + filename + '_main' + '.svg' + fname = "%s%s/%s_main.svg" % (path, stats.values['user'], filename) + print("[*] Output saved to ", fname) else: - fname = path + stats.values['user'] + '/' + filename + "_" + cat + '.svg' + fname = "%s%s/%s_%smain.svg" % (path, stats.values['user'], + filename, cat) + print("[*] Output saved to ", fname) graph_obj.render_to_file(fname) # Gets a drawable object argument and renders a PNG image of it. def draw_png(graph_obj): if cat is None: - fname = path + stats.values['user'] + '/' + filename + '_main' + '.png' + fname = "%s%s/%s_main.png" % (path, stats.values['user'], filename) else: - fname = path + stats.values['user'] + '/' + filename + "_" + cat + '.png' + fname = "%s%s/%s_%smain.png" % (path, stats.values['user'], + filename, cat) graph_obj.render_to_png(fname) @@ -60,7 +64,7 @@ def draw_bar(output_json, title): # Generates CSV report for the user from the dictionary passed def save_csv(output_json): global csv_init, cat - fname = path + stats.values['user'] + '/' + filename + '_main.csv' + fname = "%s%s/%s_main.csv" % (path, stats.values['user'], filename) fout = open(fname, 'a') csvw = csv.writer(fout) @@ -91,6 +95,7 @@ def save_csv(output_json): csvw.writerows(data) fout.close() + def show_gource(unicode_json): # Thanks Ralph. Color codes taken from fedmsg2gource @@ -101,7 +106,7 @@ def show_gource(unicode_json): colors = colors * n_wraps color_lookup = dict(zip(procs, colors)) - fname = path + stats.values['user'] + '/' + filename + '_main.gource' + fname = "%s%s/%s_main.gource" % (path, stats.values['user'], filename) fout = open(fname, 'w') for activity in unicode_json['raw_messages']: try: @@ -112,16 +117,17 @@ def show_gource(unicode_json): fout.write(u"%i|%s|A|%s|%s\n" % ( activity['timestamp'], user, - activity['topic'].split('.')[4] + " - "+ activity['topic'].split('.')[3], + activity['topic'].split('.')[4] + " - " + activity['topic'].split('.')[3], color_lookup[activity['topic'].split('.')[3]], )) fout.close() - os.system("cat " + fname + " | gource --log-format custom --highlight-user " + os.system("cat " + fname + " |gource --log-format custom --highlight-user " + stats.values['user'] + " -c 0.5 -") + # Saves category-wise text report of a user. def save_text_log(unicode_json): - fname = path + stats.values['user'] + '/' + filename + '_main.txt' + fname = "%s%s/%s_main.txt" % (path, stats.values['user'], filename) fout = open(fname, 'w') # Category-wise Log fout.write("\n\n*** Category-wise activities ***\n\n") diff --git a/stats.py b/stats.py index a7264af..fd9f19f 100644 --- a/stats.py +++ b/stats.py @@ -42,7 +42,7 @@ def return_json(): total_pages = 1 # Only pull the values from datagrepper if it's the first run - if len(unicode_json) == 0 or unicode_json['arguments']['users'][0]!=values['user']: + if len(unicode_json) == 0 or unicode_json['arguments']['users'][0] != values['user']: print('[*] Grabbing datagrepper values for user ' + values['user'] + '..') # If the user is set as all, we filter it using the provided category, @@ -96,6 +96,7 @@ def return_categories(): # Given a category, looks for subcategories in the category and returns a # sub-category counter. + def return_subcategories(category): subcat_list = list() subcategories = Counter() From 08075d0031ca4bacc6097c44b940ead154922d1a Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Aug 14 2016 15:25:16 +0000 Subject: [PATCH 9/12] Updated README --- diff --git a/README.md b/README.md index 748dcbd..c9fbaf1 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,10 @@ One-liner uses the classic argument parsing method to generate output. This is u * Takes a single word string input. This will define the output file name. This option is to be combined with the `--mode/-m` argument. Please note that this option **DOES NOT** require an extension type. For instance, if you need an SVG output with the name `nobody.svg`, the --output flag should be set as `nobody` and not `nobody.svg`. the default value is `stats`. +`--group / -g` + +* Takes a single word string input. This will take a group name of FAS as an argument. Useful for pulling statistics of an entire group. Please note that this requires authentication and the credentials are to be put in `fas_credentials.cfg` file. Not to be combined with `--user / -u` + `--logging / -l` * Stores the value as `True` if called, does not require any argument. When logging is set, all the logs from start to end / mentioned weeks will be pulled from datagrepper and dumped into a text file according to the naming convention. From 264dd090b232ea43552d93c29173fa6bede9efcc Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Aug 15 2016 07:32:58 +0000 Subject: [PATCH 10/12] Added final intern script --- diff --git a/scripts/final_intern_stats.py b/scripts/final_intern_stats.py new file mode 100644 index 0000000..609d4f0 --- /dev/null +++ b/scripts/final_intern_stats.py @@ -0,0 +1,27 @@ +import os + + +def main(): + fp = open('interns.txt') + interns = [user.strip('\n') for user in open('interns.txt')] + print interns + + for intern in interns: + if not os.path.exists(intern): + os.makedirs(intern) + # CSV MAIN FILE + os.system('python main.py -u %s -s 05/23/2016 -e 08/15/2016 -m csv -o stats_main' %( + intern)) + # Category-wise and Pagure + os.system('python main.py -u %s -s 05/23/2016 -e 08/15/2016 -m svg -o %s/%s -c pagure' %( + intern, intern, intern)) + # Markdown Report + os.system('python main.py -u %s -s 05/23/2016 -e 08/15/2016 -m markdown -o %s/README.md' %( + intern, intern)) + # User Specific + if intern == 'dhanvi': + os.system('python main.py -u %s -s 05/23/2016 -e 08/15/2016 -m svg -o %s/%s -c copr' %( + intern, intern, intern)) + +if __name__ == '__main__': + main() From 109ea407e98a0ba269c530354f7552c40a7f65ff Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Aug 15 2016 08:02:42 +0000 Subject: [PATCH 11/12] Fixed encoding bug --- diff --git a/output.py b/output.py index 8a5154e..f23eb45 100644 --- a/output.py +++ b/output.py @@ -144,8 +144,11 @@ def save_text_log(unicode_json): category.capitalize() + " **\n") flag = False - fout.write("* " + fedmsg.meta.msg2subtitle(activity).encode( - 'ascii', errors="ignore") + "\n") + try: + fout.write("* " + fedmsg.meta.msg2subtitle(activity).encode( + 'utf-8') + "\n") + except AttributeError: + pass fout.write("\nTotal Entries in category : " + str(actcount) + "\n") fout.write("\nPercentage participation in category : " + str(round(100 * actcount / @@ -206,8 +209,11 @@ def save_markdown(unicode_json): category.capitalize() + "\n") flag = False - fout.write("* " + fedmsg.meta.msg2subtitle(activity).encode( - 'ascii', errors='ignore') + "\n") + try: + fout.write("* " + fedmsg.meta.msg2subtitle(activity).encode( + 'utf-8', errors='ignore') + "\n") + except AttributeError: + pass fout.write("\n* **Total Entries in category :** " + str(actcount) + "\n") fout.write("\n* **Percentage participation in category :** " + From 4959c4b7d089085161339cc3487c5a76f45f8b74 Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Aug 15 2016 15:30:58 +0000 Subject: [PATCH 12/12] Updated documentation --- diff --git a/README.md b/README.md index c9fbaf1..290f3a5 100644 --- a/README.md +++ b/README.md @@ -19,15 +19,15 @@ One-liner uses the classic argument parsing method to generate output. This is u `--interactive / -i` -* Launches the tool in interactive mode. Does not require any further arguments. +* Launches the tool in interactive mode. Does not require any further arguments. Some of the argparse variables are yet to be implemented. `--user / -u` -* Takes any FAS Username as argument. There is no default value and the tool will throw an error if this argument is left blank/not used. +* Takes any FAS Username as argument. There is no default value and the tool will throw an error if this argument is left blank/not used. This argument can also take the value `all`. If `user` is set as `all`, then the unfiltered data is pulled from datagrepper. `--start / -s` -* Takes a date as input in the format `MM/DD/YYYY` as input, that determines the starting date for which the data is required. This will be internally converted to the epoch time. +* Takes a date as input in the format `MM/DD/YYYY` as input, that determines the starting date for which the data is required. This will be internally converted to the epoch time. If this parameter is set along with `--end`, `--delta` is ignored. `--end / -e` @@ -35,7 +35,7 @@ One-liner uses the classic argument parsing method to generate output. This is u `--weeks / -w` -* Takes an integer value to represent number of weeks. Converts it into timedelta. (1week = 604,800 seconds). Default value is 1. This values is ignored if the `--start` and `--end` values are set. +* Takes an integer value to represent number of weeks. Converts it into time-delta. (1week = 604,800 seconds). Default value is 1. This values is ignored if the `--start` and `--end` values are set. `--mode / -m` @@ -87,6 +87,10 @@ This will create `stats.svg` in your `$pwd`. `python main.py --user=foo --mode=png --output=foo_stats` +* Generate data between specific dates and locally save as JSON + +`python main.py --user=foo --mode=json --start=05/23/2016 --end=05/28/2016` + #### Basic Troubleshooting : Please take a look at this [blog-post](https://sachinwrites.xyz/2016/05/28/getting-fedstats-gsoc-production-ready/).