From 9c2642e93fdcce6386245e388205a4fead13c3e0 Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jun 04 2016 19:32:17 +0000 Subject: [PATCH 1/3] Support for categories --- diff --git a/main.py b/main.py index d6faa8f..aeb5eb8 100644 --- a/main.py +++ b/main.py @@ -5,8 +5,8 @@ import os import fedmsg import argparse import fedmsg.meta -from stats import stats -from output import draw +import stats +import output from termcolor import colored @@ -17,49 +17,48 @@ def main(): # Argument Parser initialization parser = argparse.ArgumentParser(description='Fedora GSoC stats gatherer') - parser.add_argument('--user', '-u', help='FAS username') - parser.add_argument('--weeks', '-w', help='Time in weeks', default=1) - parser.add_argument('--mode', '-m', help="Type of Output", default='text') - parser.add_argument('--output', '-o', help="Output name", default='stats') - parser.add_argument('--interactive', '-i', help="Enable interactive mode", - action='store_true') + parser.add_argument('--user', help='FAS username') + parser.add_argument('--weeks', help='Time in weeks', default=1) + parser.add_argument('--mode', help="Type of Output", default='text') + parser.add_argument('--output', help="Output name", default='stats') + parser.add_argument('--category', help="Category for graphs", default=None) + parser.add_argument('--interactive', '-i', help="Enable interactive mode", action='store_true') args = parser.parse_args() # Object inits and argument processing - userstats = stats() - output = draw() if args.interactive: - userstats.values['user'] = str(raw_input("Enter FAS Username : ")) - userstats.values['delta'] = 604800 * int(raw_input("Number of weeks stats \ -required for : ")) + stats.values['user'] = str(raw_input("Enter FAS Username : ")) + stats.values['delta'] = 604800 * int(raw_input("Number of weeks stats required for : ")) output.mode = str(raw_input("Type of output : ")) output.filename = str(raw_input("Output file : ")) elif args.user is None: - print(colored("[!] ", 'red') + "FAS Username is required.") + print(colored("[!] ", 'red') + "Username is required. Use -h for help") return 1 else: - userstats.values['user'] = str(args.user) - userstats.values['delta'] = int(args.weeks) * 604800 + stats.values['user'] = str(args.user) + stats.values['delta'] = int(args.weeks) * 604800 + stats.category = args.category output.mode = args.mode output.filename = args.output # For json and text output, we need the JSON rather than the categories if output.mode == 'svg' or output.mode == 'png': - out_obj = userstats.return_categories() + draw_obj = stats.return_categories() # To handle user with no activity - if len(out_obj) == 0: + if len(draw_obj) == 0: print ('[!] No activity found for user ' + str(args.user)) return 1 + elif args.mode.lower() == 'json' or args.mode.lower() == 'text': - out_obj = userstats.return_json() - if out_obj['total'] == 0: + draw_obj = stats.return_json() + if draw_obj['total'] == 0: print ('[!] No activity found for user ' + str(args.user)) return 1 - title = "Category distribution for user " + str(args.user) - output.show_output(out_obj, title) + # output.show_category_output(draw_obj, str(stats.values['user']), 'bar') + stats.return_interactions(['issue','pull-request']) if __name__ == '__main__': - main() + main() diff --git a/output.py b/output.py index b77715b..6d32c71 100644 --- a/output.py +++ b/output.py @@ -1,55 +1,90 @@ from __future__ import absolute_import from __future__ import print_function +from stats import * import pygal +import stats import json import os -from stats import * -class draw: +mode = 'text' +filename = 'stats' +category_json = None +subcategory_json = None + +def draw_svg(graph_obj): + fname = filename + '.svg' + graph_obj.render_to_file(fname) + os.system('firefox '+fname) + +def draw_category_png(graph_obj): + fname = filename + '.png' + graph_obj.render_to_png(filename=fname) - def __init__(self): - self.mode = 'text' - self.filename = 'stats' +def draw_pie(output_json, title): + pie_chart = pygal.Pie(inner_radius=0.4) + pie_chart.title = str(title) + for key in output_json: + percent = output_json[key] / float(sum(output_json.values())) * 100 + pie_chart.add(str(key), round(percent, 2)) + return pie_chart - def draw_svg(self, graph_obj): - filename = self.filename + '.svg' - graph_obj.render_to_file(filename) - os.system('firefox '+filename) +def draw_bar(output_json, title): + bar_chart = pygal.Bar() + bar_chart.title = str(title) + for key in output_json: + bar_chart.add(str(key), output_json[key]) + return bar_chart - def draw_png(self, graph_obj): - fname = self.filename + '.png' - graph_obj.render_to_png(filename=fname) +def save_text(unicode_json, username): + fname = filename + '.txt' + fout = open(fname, 'w') - def draw_pie(self, input_json, title): - pie_chart = pygal.Pie(inner_radius=0.4) - pie_chart.title = str(title) - for key in input_json: - percent = input_json[key] / float(sum(input_json.values())) * 100 - pie_chart.add(str(key), round(percent, 2)) - return pie_chart + # Entire Log Write + fout.write("*****Full log for user " + username + "*****\n\n\n") + for activity in unicode_json['raw_messages']: + fout.write(fedmsg.meta.msg2subtitle(activity)+"\n") - def show_logs(self, unicode_json): + # Category-wise Log + fout.write("\n\n*****Category-wise activities*****\n\n") + for category in stats.return_categories(): for activity in unicode_json['raw_messages']: - print(fedmsg.meta.msg2subtitle(activity)) - - def save_json(self, unicode_json): - filename = self.filename + '.json' - try: - with open(filename, 'w') as outfile: - json.dump(unicode_json, outfile) - except IOError: - print("[!] Could not write into directory. Check Permissions") - - def show_output(self, input_json, title): - print('[*] Readying Output..') - if self.mode.lower() == 'svg': - temp_obj = self.draw_pie(input_json, title) - self.draw_svg(temp_obj) - elif self.mode.lower() == 'png': - temp_obj = self.draw_pie(input_json, title) - self.draw_png(temp_obj) - elif self.mode.lower() == 'json': - self.save_json(input_json) - elif self.mode.lower() == 'text': - self.show_logs(input_json) + if category == activity['topic'].split('.')[3]: + fout.write() + + +def save_json(unicode_json): + filename = filename + '.json' + try: + with open(filename, 'w') as outfile: + json.dump(unicode_json, outfile) + except IOError: + print("[!] Could not write into directory. Check Permissions") + +def generate_graph(output_json, username, gtype): + print('[*] Readying Output..') + + if mode.lower() == 'svg': + if gtype == 'pie': + graph_obj = draw_pie(output_json, username) + elif gtype == 'bar': + graph_obj = draw_bar(output_json, username) + draw_svg(graph_obj) + + elif mode.lower() == 'png': + graph_obj = draw_pie(output_json, username) + draw_category_png(graph_obj) + + elif mode.lower() == 'json': + save_json(output_json) + + elif mode.lower() == 'text': + save_text(output_json, username) + +def show_subcategory_output(subcategory_json, username, gtype): + subcategory_json = subcategory_json + self.generate_graph(subcategory_json, username, gtype) + +def show_category_output(category_json, username, gtype): + category_json = category_json + generate_graph(category_json, username, gtype) diff --git a/stats.py b/stats.py index 50e43bb..7a0d726 100644 --- a/stats.py +++ b/stats.py @@ -7,29 +7,65 @@ import requests from collections import Counter -class stats: - def __init__(self): - self.values = dict() - self.values['user'] = None - self.values['delta'] = 604800 - self.values['rows_per_page'] = 100 - self.values['not_category'] = 'meetbot' - self.baseurl = "https://apps.fedoraproject.org/datagrepper/raw" - - def return_json(self): - print('[*] Grabbing datagrepper values..') - response = requests.get(self.baseurl, params=self.values) - unicode_json = json.loads(response.text) - return unicode_json - - def return_categories(self): - cat_list = list() - categories = Counter() - unicode_json = self.return_json() - print("[*] Identifying Categories..") - for activity in unicode_json['raw_messages']: - # Split the topic using . param , extract the 4th word and append - cat_list.append(activity['topic'].split('.')[3]) - for category in cat_list: - categories[category] += 1 - return categories +values = dict() +values['user'] = None +values['delta'] = 604800 +values['rows_per_page'] = 100 +values['not_category'] = 'meetbot' +category = None +baseurl = "https://apps.fedoraproject.org/datagrepper/raw" + +def return_user(): + return values['user'] + +def return_json(): + print('[*] Grabbing datagrepper values..') + response = requests.get(baseurl, params=values) + unicode_json = json.loads(response.text) + return unicode_json + +def return_categories(): + cat_list = list() + categories = Counter() + unicode_json = return_json() + print("[*] Identifying Categories..") + for activity in unicode_json['raw_messages']: + # Split the topic using . param , extract the 4th word and append + cat_list.append(activity['topic'].split('.')[3]) + for category in cat_list: + categories[category] += 1 + return categories + +def return_subcategories(category): + unicode_json = return_json() + subcat_list = list() + subcategories = Counter() + print("[*] Identifying sub-categories..") + for activity in unicode_json['raw_messages']: + if category == activity['topic'].split('.')[3]: + subcat_list.append(activity['topic'].split('.')[4]) + + for subcategory in subcat_list: + subcategories[subcategory] += 1 + return subcategories + + +def return_interactions(subcategories): + unicode_json = return_json() + interaction_dict = dict() + interaction_list = list() + + # Initializing the dictionary + for object in subcategories: + interaction_dict[object] = [] + + # Gathering sub-sub-categories + for activity in unicode_json['raw_messages']: + for object in subcategories: + if object == activity['topic'].split('.')[4] and activity['topic'].split('.')[5]: + interaction_dict[object].append(activity['topic'].split('.')[5]) + + # Changing list to a counter + for key in interaction_dict: + interaction_dict[key] = Counter(interaction_dict[key]) + return interaction_dict From f5dc366088bb162de7415594aa4e49a4bbac97dc Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jun 04 2016 21:09:14 +0000 Subject: [PATCH 2/3] Visualization of sub categories --- diff --git a/main.py b/main.py index aeb5eb8..e3ab3e7 100644 --- a/main.py +++ b/main.py @@ -26,15 +26,17 @@ def main(): args = parser.parse_args() # Object inits and argument processing - if args.interactive: stats.values['user'] = str(raw_input("Enter FAS Username : ")) stats.values['delta'] = 604800 * int(raw_input("Number of weeks stats required for : ")) + stats.category = str(raw_input("Enter category : ")) output.mode = str(raw_input("Type of output : ")) output.filename = str(raw_input("Output file : ")) + elif args.user is None: print(colored("[!] ", 'red') + "Username is required. Use -h for help") return 1 + else: stats.values['user'] = str(args.user) stats.values['delta'] = int(args.weeks) * 604800 @@ -45,6 +47,8 @@ def main(): # For json and text output, we need the JSON rather than the categories if output.mode == 'svg' or output.mode == 'png': draw_obj = stats.return_categories() + draw_obj2 = stats.return_subcategories(stats.category) + interactions = stats.return_interactions(draw_obj2) # To handle user with no activity if len(draw_obj) == 0: print ('[!] No activity found for user ' + str(args.user)) @@ -52,13 +56,19 @@ def main(): elif args.mode.lower() == 'json' or args.mode.lower() == 'text': draw_obj = stats.return_json() + # To handle user with no activity if draw_obj['total'] == 0: print ('[!] No activity found for user ' + str(args.user)) return 1 - # output.show_category_output(draw_obj, str(stats.values['user']), 'bar') - stats.return_interactions(['issue','pull-request']) - + output.generate_graph(draw_obj, "Topic distribution of " + str(stats.values['user']), 'pie') + output.generate_graph(draw_obj2, "Category: " + str(stats.category).capitalize()\ + + "\nUser: " + str(stats.values['user']), 'bar') + + if interactions != 1: + for keys in interactions: + output.generate_graph(interactions[keys], "Interaction with "+str(keys)+"\nCategory: "\ + + str(stats.category).capitalize(), 'pie') if __name__ == '__main__': main() diff --git a/output.py b/output.py index 6d32c71..f47e037 100644 --- a/output.py +++ b/output.py @@ -11,18 +11,20 @@ mode = 'text' filename = 'stats' category_json = None subcategory_json = None +count = 0 def draw_svg(graph_obj): - fname = filename + '.svg' + global count + fname = filename + str(count) + '.svg' graph_obj.render_to_file(fname) - os.system('firefox '+fname) + os.system('firefox ' + fname) def draw_category_png(graph_obj): - fname = filename + '.png' + fname = filename + str(count) + '.png' graph_obj.render_to_png(filename=fname) def draw_pie(output_json, title): - pie_chart = pygal.Pie(inner_radius=0.4) + pie_chart = pygal.Pie(inner_radius=0.4, width=500, height=500) pie_chart.title = str(title) for key in output_json: percent = output_json[key] / float(sum(output_json.values())) * 100 @@ -30,21 +32,21 @@ def draw_pie(output_json, title): return pie_chart def draw_bar(output_json, title): - bar_chart = pygal.Bar() + bar_chart = pygal.Bar(width=500, height=500) bar_chart.title = str(title) for key in output_json: bar_chart.add(str(key), output_json[key]) return bar_chart def save_text(unicode_json, username): - fname = filename + '.txt' + fname = filename + str(count) + '.txt' fout = open(fname, 'w') # Entire Log Write fout.write("*****Full log for user " + username + "*****\n\n\n") for activity in unicode_json['raw_messages']: fout.write(fedmsg.meta.msg2subtitle(activity)+"\n") - +''' # Category-wise Log fout.write("\n\n*****Category-wise activities*****\n\n") for category in stats.return_categories(): @@ -52,9 +54,9 @@ def save_text(unicode_json, username): if category == activity['topic'].split('.')[3]: fout.write() - +''' def save_json(unicode_json): - filename = filename + '.json' + filename = filename + str(count) + '.json' try: with open(filename, 'w') as outfile: json.dump(unicode_json, outfile) @@ -62,8 +64,9 @@ def save_json(unicode_json): print("[!] Could not write into directory. Check Permissions") def generate_graph(output_json, username, gtype): + global count print('[*] Readying Output..') - + count += 1 if mode.lower() == 'svg': if gtype == 'pie': graph_obj = draw_pie(output_json, username) @@ -80,11 +83,3 @@ def generate_graph(output_json, username, gtype): elif mode.lower() == 'text': save_text(output_json, username) - -def show_subcategory_output(subcategory_json, username, gtype): - subcategory_json = subcategory_json - self.generate_graph(subcategory_json, username, gtype) - -def show_category_output(category_json, username, gtype): - category_json = category_json - generate_graph(category_json, username, gtype) diff --git a/stats.py b/stats.py index 7a0d726..265bbc5 100644 --- a/stats.py +++ b/stats.py @@ -14,11 +14,13 @@ values['rows_per_page'] = 100 values['not_category'] = 'meetbot' category = None baseurl = "https://apps.fedoraproject.org/datagrepper/raw" +unicode_json={} def return_user(): return values['user'] def return_json(): + global unicode_json print('[*] Grabbing datagrepper values..') response = requests.get(baseurl, params=values) unicode_json = json.loads(response.text) @@ -37,7 +39,6 @@ def return_categories(): return categories def return_subcategories(category): - unicode_json = return_json() subcat_list = list() subcategories = Counter() print("[*] Identifying sub-categories..") @@ -51,7 +52,6 @@ def return_subcategories(category): def return_interactions(subcategories): - unicode_json = return_json() interaction_dict = dict() interaction_list = list() @@ -62,8 +62,12 @@ def return_interactions(subcategories): # Gathering sub-sub-categories for activity in unicode_json['raw_messages']: for object in subcategories: - if object == activity['topic'].split('.')[4] and activity['topic'].split('.')[5]: - interaction_dict[object].append(activity['topic'].split('.')[5]) + try: + if object == activity['topic'].split('.')[4] and activity['topic'].split('.')[5]: + interaction_dict[object].append(activity['topic'].split('.')[5]) + except IndexError: + print("[!] That category doesn't have any more interactions!") + return 1 # Changing list to a counter for key in interaction_dict: From 42d7e79606c8a327c2fe6f2a7fef4427e5b7b41b Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jun 04 2016 21:13:27 +0000 Subject: [PATCH 3/3] Added text files to gitignore --- diff --git a/.gitignore b/.gitignore index 51c5824..60cb08a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,5 @@ *.*~ *.png *.svg - +*.txt *.json