From bb0871c593f53acc5d9af3573f790164bb474779 Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jun 12 2016 16:58:43 +0000 Subject: [PATCH 1/6] Added user=all argument --- diff --git a/main.py b/main.py index c701336..afa7b7b 100644 --- a/main.py +++ b/main.py @@ -71,7 +71,7 @@ def main(): # Check if a category input was given and generate the specific category graph if not stats.category is None : output.generate_graph(draw_obj2, "Category: " + stats.category.capitalize()\ - + "\nUser: " + stats.values['user'], stats.category, 'bar') + + "\nUser: " + args.user, stats.category, 'bar') # Check if the sub-sub-category exists and generate it's graph if not None in list(interactions.keys()): @@ -87,7 +87,7 @@ def main(): if draw_obj['total'] == 0: print (colored("[!] ", 'red') + 'No activity found for user ' + str(args.user)) return 1 - output.generate_graph(draw_obj, stats.values['user']) + output.generate_graph(draw_obj, args.user) diff --git a/stats.py b/stats.py index 1b8a690..6655c87 100644 --- a/stats.py +++ b/stats.py @@ -13,7 +13,7 @@ values['delta'] = 604800 values['rows_per_page'] = 100 values['not_category'] = 'meetbot' values['page'] = 1 -category = None +category = '' weeks = 0 baseurl = "https://apps.fedoraproject.org/datagrepper/raw" unicode_json={} @@ -22,23 +22,36 @@ unicode_json={} def return_json(): global unicode_json total_pages = 1 - if category is not None : - values['category'] = category + + # Only pull the values from datagrepper if it's the first run if len(unicode_json) == 0: print('[*] Grabbing datagrepper values..') - response = requests.get(baseurl, params=values) + + # If the user is set as all, we filter it using the provided category, if any + if category != '' and values['user'] == 'all': + values['category'] = category + # If the user value is passed as all, remove it from the dict and pass arguments + if values['user'] == 'all': + temp_dict = values + del(temp_dict['user']) + response = requests.get(baseurl, params=temp_dict) + else: + response = requests.get(baseurl, params=values) unicode_json = json.loads(response.text) total_pages = unicode_json['pages'] print ("Total pages found : " + str(total_pages)) + + # If multiple pages exist, get them all. while total_pages > 1: values['page'] += 1 print(" [*] Pulling data from page " + str(values['page'])) response = requests.get(baseurl, params=values) paginated_json = json.loads(response.text) + # Pull data from multiple pages and append them to the main JSON for activity in paginated_json['raw_messages'] : unicode_json['raw_messages'].append(activity) total_pages -= 1 - + print(unicode_json) return unicode_json # Analyzes the JSON and return categories present as a list. From 14580d1e055fc489d972d7d47464d774baef7965 Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jun 13 2016 14:48:32 +0000 Subject: [PATCH 2/6] Automation ready --- diff --git a/automate.py b/automate.py new file mode 100644 index 0000000..39e5957 --- /dev/null +++ b/automate.py @@ -0,0 +1,11 @@ +import os + +def main(): + + f = open('users.txt') + lines = [line.rstrip('\n') for line in open('users.txt')] + print lines + + for user in lines: + os.system('python main.py -u ' + user + ' -s 05/28/2016 -e 06/05/2016 -m csv') +main() diff --git a/category_scraper.py b/category_scraper.py new file mode 100644 index 0000000..470ee8d --- /dev/null +++ b/category_scraper.py @@ -0,0 +1,11 @@ +from bs4 import BeautifulSoup +from urllib import urlopen + +page = urlopen ('https://fedora-fedmsg.readthedocs.io/en/latest/topics.html') +soup = BeautifulSoup (page.read().decode('ascii', 'ignore'), 'html.parser') +categorylist = list() +for h3 in soup.findAll('h3'): + if len(h3.text.split('.')) > 2 and h3.text.split('.') not in categorylist : + categorylist.append(h3.text.split('.')) + +print categorylist diff --git a/main.py b/main.py index afa7b7b..eb7f524 100644 --- a/main.py +++ b/main.py @@ -5,12 +5,15 @@ from six.moves import input import os import fedmsg import argparse +import calendar import fedmsg.meta import stats import output from termcolor import colored + + def main(): # fedmsg config config = fedmsg.config.load_config() @@ -27,6 +30,8 @@ def main(): parser.add_argument('--mode', '-m', help="Type of Output", default='text') parser.add_argument('--output', '-o', help="Output name", default='stats') parser.add_argument('--category', '-c', help="Category for graphs", default=None) + parser.add_argument('--start', '-s', help="Start Date", default='') + parser.add_argument('--end','-e', help = "End Date", default='' ) parser.add_argument('--interactive', '-i', help="Enable interactive mode", action='store_true') args = parser.parse_args() args.output = args.user @@ -50,6 +55,8 @@ def main(): stats.values['user'] = str(args.user).lower() stats.values['delta'] = int(args.weeks) * 604800 stats.category = args.category + stats.start = args.start + stats.end = args.end stats.weeks = int(args.weeks) output.mode = args.mode.lower() output.filename = args.output.lower() diff --git a/output.py b/output.py index 07e4242..0fb6ee2 100644 --- a/output.py +++ b/output.py @@ -33,7 +33,7 @@ def draw_png(graph_obj): fname = filename + '_main' + '.png' else : fname = filename + "_" + cat + '.png' - graph_obj.render_to_png(filename=fname) + graph_obj.render_to_png(fname) # Generates a drawable pie chart object from a dictionary passed. def draw_pie(output_json, title): @@ -62,9 +62,9 @@ def save_csv(output_json): # Write the dates into CSV if not csv_init : csvw.writerows ( - [['Start Date : ', date.today() - timedelta(days=stats.weeks*7)], - ['End Date : ', date.today()], - ['']] + [['Start Date : ', stats.start], + ['End Date : ', stats.end], + ['']] ) csv_init = True @@ -78,6 +78,7 @@ def save_csv(output_json): else: data.append([stats.values['user'], key.capitalize(), output_json[key], str(percent)+'%']) + # Insert blank lines and total data.append(['']) data.append(['', 'Total : ', sum(output_json.values())]) data.append(['']) @@ -111,7 +112,6 @@ def save_text(unicode_json): def save_markdown(unicode_json): fname = filename + '_main.md' fout = open(fname, 'w') - # Category-wise Log, markdown ready fout.write("\n\n### Category-wise activities\n\n") for category in stats.return_categories(): @@ -131,7 +131,7 @@ def save_markdown(unicode_json): fout.close() render_report(fname) -# WIP - Renders HTML from markdown +# WIP - Renders HTML from markdown, invoked my markdown function atm. def render_report(fname): grip.export(fname, title="Summer Coding Statistics") diff --git a/stats.py b/stats.py index 6655c87..fc96ce5 100644 --- a/stats.py +++ b/stats.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import print_function import fedmsg import fedmsg.meta +import calendar import json import requests from collections import Counter @@ -13,11 +14,22 @@ values['delta'] = 604800 values['rows_per_page'] = 100 values['not_category'] = 'meetbot' values['page'] = 1 +values['size'] = 'small' category = '' +start = '' +end = '' weeks = 0 baseurl = "https://apps.fedoraproject.org/datagrepper/raw" unicode_json={} +def return_epoch(time): + if time == '': + return '' + tup = map(int,time.split('/')) + l = (tup[2], tup[0], tup[1], 0, 0, 0) + epochs = calendar.timegm(l) + return (int(epochs)) + # Checks if unicode_json is empty, pulls datagrepper values and returns the json def return_json(): global unicode_json @@ -30,9 +42,13 @@ def return_json(): # If the user is set as all, we filter it using the provided category, if any if category != '' and values['user'] == 'all': values['category'] = category + if start != '' and end != '' : + values['start'] = return_epoch(start) + values['end'] = return_epoch(end) + del(values['delta']) # If the user value is passed as all, remove it from the dict and pass arguments if values['user'] == 'all': - temp_dict = values + temp_dict = dict(values) del(temp_dict['user']) response = requests.get(baseurl, params=temp_dict) else: @@ -51,7 +67,6 @@ def return_json(): for activity in paginated_json['raw_messages'] : unicode_json['raw_messages'].append(activity) total_pages -= 1 - print(unicode_json) return unicode_json # Analyzes the JSON and return categories present as a list. From a69ba72260e695dafb18f2986fab090f2f1d447e Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jun 13 2016 14:55:14 +0000 Subject: [PATCH 3/6] Minor csv bug fix --- diff --git a/main.py b/main.py index eb7f524..e498488 100644 --- a/main.py +++ b/main.py @@ -76,7 +76,7 @@ def main(): interactions = stats.return_interactions(draw_obj2) # Check if a category input was given and generate the specific category graph - if not stats.category is None : + if stats.category != '' : output.generate_graph(draw_obj2, "Category: " + stats.category.capitalize()\ + "\nUser: " + args.user, stats.category, 'bar') From 1c7d2859d285a1140c2d08a0108a086cde142ae0 Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jun 14 2016 19:20:53 +0000 Subject: [PATCH 4/6] Code cleanup and added pycon_us script --- diff --git a/automate.py b/automate.py deleted file mode 100644 index 39e5957..0000000 --- a/automate.py +++ /dev/null @@ -1,11 +0,0 @@ -import os - -def main(): - - f = open('users.txt') - lines = [line.rstrip('\n') for line in open('users.txt')] - print lines - - for user in lines: - os.system('python main.py -u ' + user + ' -s 05/28/2016 -e 06/05/2016 -m csv') -main() diff --git a/main.py b/main.py index e498488..f2636ca 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,7 @@ from __future__ import print_function from __future__ import absolute_import +from termcolor import colored from six.moves import input - import os import fedmsg import argparse @@ -9,9 +9,6 @@ import calendar import fedmsg.meta import stats import output -from termcolor import colored - - def main(): @@ -19,20 +16,21 @@ def main(): config = fedmsg.config.load_config() fedmsg.meta.make_processors(**config) - #Initializing to None to prevent errors while generating multiple reports + # Initializing to None to prevent errors while generating multiple reports stats.unicode_json = {} i_count = 0 # Argument Parser initialization - parser = argparse.ArgumentParser(description='Summer Coding stats gatherer') + parser = argparse.ArgumentParser(description='Fedora Statistics Gatherer') parser.add_argument('--user', '-u', help='FAS username') - parser.add_argument('--weeks','-w', help='Time in weeks', default=1) + parser.add_argument('--weeks', '-w', help='Time in weeks', default=1) parser.add_argument('--mode', '-m', help="Type of Output", default='text') parser.add_argument('--output', '-o', help="Output name", default='stats') - parser.add_argument('--category', '-c', help="Category for graphs", default=None) + parser.add_argument('--category', '-c', help="Sub Category", default='') parser.add_argument('--start', '-s', help="Start Date", default='') - parser.add_argument('--end','-e', help = "End Date", default='' ) - parser.add_argument('--interactive', '-i', help="Enable interactive mode", action='store_true') + parser.add_argument('--end', '-e', help="End Date", default='') + parser.add_argument('--interactive', '-i', help="Enable interactive mode", + action='store_true') args = parser.parse_args() args.output = args.user @@ -40,7 +38,7 @@ def main(): if args.interactive: stats.values['user'] = str(input("Enter FAS Username : ")).lower() stats.weeks = int(args.weeks) - stats.values['delta'] = 604800 * int(input("Number of weeks stats required for : ")) + stats.values['delta'] = 604800 * int(input("Number of weeks : ")) stats.category = str(input("Enter category : ")).lower() output.mode = str(input("Type of output : ")).lower() output.filename = str(input("Output file : ")).lower() @@ -54,49 +52,53 @@ def main(): else: stats.values['user'] = str(args.user).lower() stats.values['delta'] = int(args.weeks) * 604800 - stats.category = args.category + stats.category = args.category.lower() stats.start = args.start stats.end = args.end stats.weeks = int(args.weeks) output.mode = args.mode.lower() output.filename = args.output.lower() - # For json and text output, we need the JSON rather than the categories + # For png and SVG, we need a drawable object to be called if output.mode in ['png', 'svg', 'csv']: # Draw object for the above mentioned categories. draw_obj = stats.return_categories() - # To handle user with no activity - if len(draw_obj) == 0: - print (colored("[!] ", 'red') + 'No activity found for user ' + args.user) + # To handle user with no activity; TO-DO -> Make this a function + if len(draw_obj) == 0: + print (colored("[!] ", 'red') + 'No activity found for user' + + args.user) return 1 # Generate the output graph objects required for calling generate_graph - output.generate_graph(draw_obj, "Topic distribution of " + stats.values['user'], None, 'pie') + output.generate_graph(draw_obj, "Topic distribution of " + + stats.values['user'], None, 'pie') draw_obj2 = stats.return_subcategories(stats.category) interactions = stats.return_interactions(draw_obj2) - # Check if a category input was given and generate the specific category graph - if stats.category != '' : - output.generate_graph(draw_obj2, "Category: " + stats.category.capitalize()\ - + "\nUser: " + args.user, stats.category, 'bar') + # Check if a category input was given and generate category graph + if stats.category != '': + output.generate_graph(draw_obj2, "Category: " + stats.category + + "\nUser: " + args.user, stats.category, 'bar') # Check if the sub-sub-category exists and generate it's graph - if not None in list(interactions.keys()): + if None not in list(interactions.keys()): for keys in interactions: i_count += 1 - output.generate_graph(interactions[keys], "Interaction with "+str(keys)+"\nCategory: "\ - + stats.category.capitalize(), stats.category +"_" + keys, 'pie') + output.generate_graph(interactions[keys], "Interaction with " + + str(keys)+"\nCategory: " + stats.category.capitalize(), + stats.category + "_" + keys, 'pie') - # If not image based outputs, check if the input matches any of the text based inputs - elif output.mode in ['json','text','markdown']: + # If not image, check if the input matches any of the text based inputs + elif output.mode in ['json', 'text', 'markdown']: draw_obj = stats.return_json() + # To handle user with no activity if draw_obj['total'] == 0: - print (colored("[!] ", 'red') + 'No activity found for user ' + str(args.user)) + print (colored("[!] ", 'red') + 'No activity found for user ' + + str(args.user)) return 1 output.generate_graph(draw_obj, args.user) - if __name__ == '__main__': main() diff --git a/output.py b/output.py index 0fb6ee2..df83668 100644 --- a/output.py +++ b/output.py @@ -18,22 +18,25 @@ csv_init = False mode = 'text' cat = None + # Gets a drawable object argument and renders an SVG Image of it. def draw_svg(graph_obj): - if cat is None : + if cat is None: fname = filename + '_main' + '.svg' - else : + else: fname = filename + "_" + cat + '.svg' graph_obj.render_to_file(fname) os.system("firefox " + fname) + # Gets a drawable object argument and renders a PNG image of it. def draw_png(graph_obj): - if cat is None : + if cat is None: fname = filename + '_main' + '.png' - else : + else: fname = filename + "_" + cat + '.png' - graph_obj.render_to_png(fname) + graph_obj.render_to_png(fname) + # Generates a drawable pie chart object from a dictionary passed. def draw_pie(output_json, title): @@ -44,6 +47,7 @@ def draw_pie(output_json, title): pie_chart.add(str(key), round(percent, 2)) return pie_chart + # Generates a drawable pie chart object from a dictionary passed. def draw_bar(output_json, title): bar_chart = pygal.Bar(width=500, height=500) @@ -52,31 +56,33 @@ def draw_bar(output_json, title): bar_chart.add(str(key), output_json[key]) return bar_chart + # Generates CSV report for the user from the dictionary passed def save_csv(output_json): - global csv_init,cat + global csv_init, cat fname = filename + '_main.csv' fout = open(fname, 'a') csvw = csv.writer(fout) # Write the dates into CSV - if not csv_init : - csvw.writerows ( - [['Start Date : ', stats.start], - ['End Date : ', stats.end], - ['']] - ) + if not csv_init: + csvw.writerows( + [['Start Date : ', stats.start], + ['End Date : ', stats.end], + ['']]) csv_init = True # Initial heading row - data = [['Username','Category', 'Activity Count', 'Percentage'],[]] + data = [['Username', 'Category', 'Activity Count', 'Percentage'], []] for key in output_json: - percent = round(output_json[key] / float(sum(output_json.values())) * 100, 2) + percent = round(output_json[key] / float(sum(output_json.values())) * + 100, 2) if cat is not None and cat.capitalize() != key.capitalize(): - data.append([stats.values['user'], cat.capitalize() + "." + \ + data.append([stats.values['user'], cat.capitalize() + "." + key.capitalize(), output_json[key], str(percent)+'%']) else: - data.append([stats.values['user'], key.capitalize(), output_json[key], str(percent)+'%']) + data.append([stats.values['user'], key.capitalize(), + output_json[key], str(percent)+'%']) # Insert blank lines and total data.append(['']) @@ -85,6 +91,7 @@ def save_csv(output_json): csvw.writerows(data) fout.close() + # Saves category-wise text report of a user. def save_text(unicode_json): fname = filename + '_main.txt' @@ -100,14 +107,15 @@ def save_text(unicode_json): actcount += 1 # Print the category once if flag is True: - fout.write("\n\n** Category : "+category.capitalize()+" **\n") + fout.write("\n\n** Category : "+category.capitalize() + " **\n") flag = False fout.write("* "+fedmsg.meta.msg2subtitle(activity)+"\n") fout.write("\nTotal Entries in category : " + str(actcount) + "\n") - fout.write("\nPercentage participation in category : " + \ - str(round(100*actcount/float(unicode_json['total']),2)) + "\n") + fout.write("\nPercentage participation in category : " + + str(round(100*actcount/float(unicode_json['total']),2)) + "\n") fout.close() + # Saves the markdown version of the text log def save_markdown(unicode_json): fname = filename + '_main.md' @@ -126,15 +134,17 @@ def save_markdown(unicode_json): flag = False fout.write("* "+fedmsg.meta.msg2subtitle(activity)+"\n") fout.write("\n* **Total Entries in category :** " + str(actcount) + "\n") - fout.write("\n* **Percentage participation in category :** " + \ - str(round(100*actcount/float(unicode_json['total']),2)) + "\n") + fout.write("\n* **Percentage participation in category :** " + + str(round(100*actcount/float(unicode_json['total']), 2)) + "\n") fout.close() render_report(fname) + # WIP - Renders HTML from markdown, invoked my markdown function atm. def render_report(fname): grip.export(fname, title="Summer Coding Statistics") + # Saves the JSON as a file. def save_json(unicode_json): fname = filename + '_main.json' @@ -144,7 +154,8 @@ def save_json(unicode_json): except IOError: print("[!] Could not write into directory. Check Permissions") -# Identifies categories, and generates drawable / file objects for all the above functions. + +# Identifies categories & generates drawable objects for the above functions. def generate_graph(output_json, title, category=None, gtype=None): global cat cat = category diff --git a/us_pycon_autostats.py b/us_pycon_autostats.py new file mode 100644 index 0000000..39e5957 --- /dev/null +++ b/us_pycon_autostats.py @@ -0,0 +1,11 @@ +import os + +def main(): + + f = open('users.txt') + lines = [line.rstrip('\n') for line in open('users.txt')] + print lines + + for user in lines: + os.system('python main.py -u ' + user + ' -s 05/28/2016 -e 06/05/2016 -m csv') +main() From 829e9e272ffe8ad6e3f2b457726bf6d49f6862d5 Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jun 21 2016 10:09:01 +0000 Subject: [PATCH 5/6] Added gource output, modified gitignore and tweaked text mode --- diff --git a/.gitignore b/.gitignore index 0cf957f..e20eb79 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ *.json *.html *.csv +*.md +*.gource diff --git a/README.md b/README.md index 8a90351..466c9c3 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A simple CLI to tool to gather statistics from [datagrepper](https://apps.fedoraproject.org/datagrepper/) ### Description -This tool will help anyone to pull statistics of any registered Fedora user with an active [FAS](https://fedoraproject.org/wiki/Account_System) account. +This tool helps pull statistics of any Fedora user with an active [FAS Account](https://fedoraproject.org/wiki/Account_System) account. ### Usage @@ -25,9 +25,17 @@ One-liner uses the classic argument parsing method to generate output. This is u * Takes any FAS Username as argument. There is no default value and the tool will throw an error if this argument is left blank/not used. +`--start / -s` + +* Takes a date as input in the format `MM/DD/YYYY` as input, that determines the starting date for which the data is required. This will be internally converted to the epoch time. + +`--end / -e` + +* Takes a date as input in the format `MM/DD/YYYY` as input, that determines the end date for which the data is required. This will be internally converted to the epoch time. + `--weeks / -w` -* Takes an integer value to represent number of weeks. Converts it into timedelta. (1week = 604,800 seconds). Default value is 1. +* Takes an integer value to represent number of weeks. Converts it into timedelta. (1week = 604,800 seconds). Default value is 1. This values is ignored if the `--start` and `--end` values are set. `--mode / -m` @@ -39,7 +47,11 @@ One-liner uses the classic argument parsing method to generate output. This is u `--output / -o` -* Takes a single word string input. This will define the output file name. This option is to be combined with the `--mode/-m` argument. Please note that this option **DOES NOT** require an extension type. For instance, if you need an SVG output with the name `nobody.svg`, the --output flag should be set as `nobody` and not `nobody.svg`. the default value is `stats` +* Takes a single word string input. This will define the output file name. This option is to be combined with the `--mode/-m` argument. Please note that this option **DOES NOT** require an extension type. For instance, if you need an SVG output with the name `nobody.svg`, the --output flag should be set as `nobody` and not `nobody.svg`. the default value is `stats`. + +`--logging / -l` + +* Stores the value as `True` if called, does not require any argument. When logging is set, all the logs from start to end / mentioned weeks will be pulled from datagrepper and dumped into a text file according to the naming convention. #### File naming convention @@ -52,7 +64,7 @@ The naming convention is as follows : * The sub-category report (i.e the Category bar-chart) is named as `_.` -* The further category interaction report (i.e The sub-categories chart) is named as `__.` +* The further category interaction report (i.e The sub-categories chart) is named as `__.` ####Examples diff --git a/category_scraper.py b/category_scraper.py index 470ee8d..dbfad72 100644 --- a/category_scraper.py +++ b/category_scraper.py @@ -1,11 +1,11 @@ from bs4 import BeautifulSoup from urllib import urlopen -page = urlopen ('https://fedora-fedmsg.readthedocs.io/en/latest/topics.html') -soup = BeautifulSoup (page.read().decode('ascii', 'ignore'), 'html.parser') +page = urlopen('https://fedora-fedmsg.readthedocs.io/en/latest/topics.html') +soup = BeautifulSoup(page.read().decode('ascii', 'ignore'), 'html.parser') categorylist = list() for h3 in soup.findAll('h3'): - if len(h3.text.split('.')) > 2 and h3.text.split('.') not in categorylist : + if len(h3.text.split('.')) > 2 and h3.text.split('.') not in categorylist: categorylist.append(h3.text.split('.')) print categorylist diff --git a/main.py b/main.py index f2636ca..2d4f055 100644 --- a/main.py +++ b/main.py @@ -11,17 +11,31 @@ import stats import output -def main(): - # fedmsg config - config = fedmsg.config.load_config() - fedmsg.meta.make_processors(**config) +def interactive_input(args): + stats.values['user'] = str(input("FAS Username (required) \t: ")).lower() + stats.values['delta'] = 604800 * int(input("Number of weeks (default : 1)\t: ")) + stats.weeks = int(args.weeks) + stats.category = str(input("Enter category (default : None)\t: ")).lower() or '' + stats.start = str(input("Start Time (MM/DD/YYYY) \t: ")) or '' + stats.end = str(input("Start Time (MM/DD/YYYY) \t: ")) or '' + output.mode = str(input("Output Type (default : text) \t: ")).lower() or 'text' + output.filename = str(input("Filename [default : $username] \t: ")).lower() or \ + stats.values['user'] - # Initializing to None to prevent errors while generating multiple reports - stats.unicode_json = {} - i_count = 0 - # Argument Parser initialization - parser = argparse.ArgumentParser(description='Fedora Statistics Gatherer') +def assign_values(args): + stats.values['user'] = str(args.user).lower() + stats.values['delta'] = int(args.weeks) * 604800 + stats.category = args.category.lower() + stats.start = args.start + stats.end = args.end + stats.weeks = int(args.weeks) + stats.log = args.log + output.mode = args.mode.lower() + output.filename = args.output.lower() + + +def add_arguments(parser): parser.add_argument('--user', '-u', help='FAS username') parser.add_argument('--weeks', '-w', help='Time in weeks', default=1) parser.add_argument('--mode', '-m', help="Type of Output", default='text') @@ -29,19 +43,29 @@ def main(): parser.add_argument('--category', '-c', help="Sub Category", default='') parser.add_argument('--start', '-s', help="Start Date", default='') parser.add_argument('--end', '-e', help="End Date", default='') + parser.add_argument('--log', '-l', help="Enable full log reporting", + action='store_true') parser.add_argument('--interactive', '-i', help="Enable interactive mode", action='store_true') + + +def main(): + # fedmsg config + config = fedmsg.config.load_config() + fedmsg.meta.make_processors(**config) + + # Initializing to None to prevent errors while generating multiple reports + stats.unicode_json = {} + i_count = 0 + + # Argument Parser initialization + parser = argparse.ArgumentParser(description='Fedora Statistics Gatherer') + add_arguments(parser) args = parser.parse_args() - args.output = args.user # Check if the argument type is interactive if args.interactive: - stats.values['user'] = str(input("Enter FAS Username : ")).lower() - stats.weeks = int(args.weeks) - stats.values['delta'] = 604800 * int(input("Number of weeks : ")) - stats.category = str(input("Enter category : ")).lower() - output.mode = str(input("Type of output : ")).lower() - output.filename = str(input("Output file : ")).lower() + interactive_input(args) # Check if the user argument exists elif args.user is None: @@ -50,52 +74,58 @@ def main(): # Else, use the argparse values. No arguments is handled by argparse. else: - stats.values['user'] = str(args.user).lower() - stats.values['delta'] = int(args.weeks) * 604800 - stats.category = args.category.lower() - stats.start = args.start - stats.end = args.end - stats.weeks = int(args.weeks) - output.mode = args.mode.lower() - output.filename = args.output.lower() + assign_values(args) # For png and SVG, we need a drawable object to be called - if output.mode in ['png', 'svg', 'csv']: + if output.mode in ['png', 'svg', 'csv'] or not stats.log and output.mode == 'text': # Draw object for the above mentioned categories. draw_obj = stats.return_categories() # To handle user with no activity; TO-DO -> Make this a function if len(draw_obj) == 0: print (colored("[!] ", 'red') + 'No activity found for user' + - args.user) + args.user) return 1 # Generate the output graph objects required for calling generate_graph output.generate_graph(draw_obj, "Topic distribution of " + - stats.values['user'], None, 'pie') + stats.values['user'], None, 'bar') draw_obj2 = stats.return_subcategories(stats.category) interactions = stats.return_interactions(draw_obj2) # Check if a category input was given and generate category graph if stats.category != '': - output.generate_graph(draw_obj2, "Category: " + stats.category + - "\nUser: " + args.user, stats.category, 'bar') + output.generate_graph( + draw_obj2, + "Category: " + + stats.category + + "\nUser: " + + args.user, + stats.category, + 'pie') # Check if the sub-sub-category exists and generate it's graph if None not in list(interactions.keys()): for keys in interactions: i_count += 1 - output.generate_graph(interactions[keys], "Interaction with " - + str(keys)+"\nCategory: " + stats.category.capitalize(), - stats.category + "_" + keys, 'pie') + output.generate_graph( + interactions[keys], + "Interaction with " + + str(keys) + + "\nCategory: " + + stats.category.capitalize(), + stats.category + + "_" + + keys, + 'bar') # If not image, check if the input matches any of the text based inputs - elif output.mode in ['json', 'text', 'markdown']: + elif output.mode in ['json', 'text', 'markdown', 'gource']: draw_obj = stats.return_json() # To handle user with no activity if draw_obj['total'] == 0: print (colored("[!] ", 'red') + 'No activity found for user ' + - str(args.user)) + str(args.user)) return 1 output.generate_graph(draw_obj, args.user) diff --git a/output.py b/output.py index df83668..e0157cf 100644 --- a/output.py +++ b/output.py @@ -5,16 +5,16 @@ import fedmsg.meta import fedmsg import stats import pygal +import math import json -import grip import csv import os # Default global variables subcategory_json = None category_json = None -filename = 'stats' -csv_init = False +filename = stats.values['user'] +csv_init = text_init = False mode = 'text' cat = None @@ -65,25 +65,25 @@ def save_csv(output_json): csvw = csv.writer(fout) # Write the dates into CSV - if not csv_init: + if not text_init and stats.end and stats.start: csvw.writerows( [['Start Date : ', stats.start], ['End Date : ', stats.end], ['']]) csv_init = True - # Initial heading row data = [['Username', 'Category', 'Activity Count', 'Percentage'], []] for key in output_json: percent = round(output_json[key] / float(sum(output_json.values())) * 100, 2) if cat is not None and cat.capitalize() != key.capitalize(): - data.append([stats.values['user'], cat.capitalize() + "." + - key.capitalize(), output_json[key], str(percent)+'%']) + data.append([stats.values['user'], + cat.capitalize() + "." + key.capitalize(), + output_json[key], + str(percent) + '%']) else: data.append([stats.values['user'], key.capitalize(), - output_json[key], str(percent)+'%']) - + output_json[key], str(percent) + '%']) # Insert blank lines and total data.append(['']) data.append(['', 'Total : ', sum(output_json.values())]) @@ -91,12 +91,38 @@ def save_csv(output_json): csvw.writerows(data) fout.close() +def show_gource(unicode_json): + + # Thanks Ralph. Color codes taken from fedmsg2gource + procs = [proc.__name__.lower() for proc in fedmsg.meta.processors] + colors = ["FFFFFF", "008F37", "FF680A", "CC4E00", + "8F0058", "8F7E00", "37008F", "7E008F"] + n_wraps = int(math.ceil(len(procs) / float(len(colors)))) + colors = colors * n_wraps + color_lookup = dict(zip(procs, colors)) + + fname = filename + '_main.gource' + fout = open(fname, 'w') + for activity in unicode_json['raw_messages']: + try: + user = list(fedmsg.meta.msg2usernames(activity))[0] + except IndexError: + user = stats.values['user'] + + fout.write(u"%i|%s|A|%s|%s\n" % ( + activity['timestamp'], + user, + activity['topic'].split('.')[4] + " - "+ activity['topic'].split('.')[3], + color_lookup[activity['topic'].split('.')[3]], + )) + fout.close() + os.system("cat " + fname + " | gource --log-format custom --highlight-user " + + stats.values['user'] + " -c 0.5 -") # Saves category-wise text report of a user. -def save_text(unicode_json): +def save_text_log(unicode_json): fname = filename + '_main.txt' fout = open(fname, 'w') - # Category-wise Log fout.write("\n\n*** Category-wise activities ***\n\n") for category in stats.return_categories(): @@ -107,12 +133,50 @@ def save_text(unicode_json): actcount += 1 # Print the category once if flag is True: - fout.write("\n\n** Category : "+category.capitalize() + " **\n") + fout.write( + "\n\n** Category : " + + category.capitalize() + + " **\n") flag = False - fout.write("* "+fedmsg.meta.msg2subtitle(activity)+"\n") + fout.write("* " + fedmsg.meta.msg2subtitle(activity).encode( + 'ascii', errors="ignore") + "\n") fout.write("\nTotal Entries in category : " + str(actcount) + "\n") fout.write("\nPercentage participation in category : " + - str(round(100*actcount/float(unicode_json['total']),2)) + "\n") + str(round(100 * actcount / + float(unicode_json['total']), 2)) + "\n") + fout.close() + + +def save_text_metrics(output_json): + global text_init + fname = filename + '_main.txt' + fout = open(fname, 'a') + # Write the dates into CSV + if not text_init and stats.end and stats.start: + fout.write( + [['Start Date : ', stats.start], + ['End Date : ', stats.end], + ['']]) + text_init = True + + # Initial heading row + data = 'Username\t\tCategory\t\tCount\t\tPercentage\n' + for key in output_json: + percent = round(output_json[key] / float(sum(output_json.values())) * + 100, 2) + if cat is not None and cat.capitalize() != key.capitalize(): + data += '%s\t\t%s\t\t%d\t\t%s\n' % ( + stats.values['user'], + cat.capitalize() + "." + key.capitalize(), + output_json[key], + str(percent) + '%') + else: + data += '%s\t\t%s\t\t%d\t\t%s\n' % ( + stats.values['user'], key.capitalize(), + output_json[key], str(percent) + '%') + # Insert blank lines and total + data += '\n\n Total : %d \n' % (sum(output_json.values())) + fout.write(data) fout.close() @@ -130,19 +194,19 @@ def save_markdown(unicode_json): actcount += 1 # Print the category once if flag is True: - fout.write("\n\n#### Category : "+category.capitalize()+"\n") + fout.write( + "\n\n#### Category : " + + category.capitalize() + + "\n") flag = False - fout.write("* "+fedmsg.meta.msg2subtitle(activity)+"\n") - fout.write("\n* **Total Entries in category :** " + str(actcount) + "\n") + fout.write("* " + fedmsg.meta.msg2subtitle(activity).encode( + 'ascii', errors='ignore') + "\n") + fout.write("\n* **Total Entries in category :** " + + str(actcount) + "\n") fout.write("\n* **Percentage participation in category :** " + - str(round(100*actcount/float(unicode_json['total']), 2)) + "\n") + str(round(100 * actcount / + float(unicode_json['total']), 2)) + "\n") fout.close() - render_report(fname) - - -# WIP - Renders HTML from markdown, invoked my markdown function atm. -def render_report(fname): - grip.export(fname, title="Summer Coding Statistics") # Saves the JSON as a file. @@ -176,10 +240,15 @@ def generate_graph(output_json, title, category=None, gtype=None): elif mode.lower() == 'json': save_json(output_json) elif mode.lower() == 'text': - save_text(output_json) + if stats.log: + save_text_log(output_json) + else: + save_text_metrics(output_json) elif mode.lower() == 'csv': save_csv(output_json) elif mode.lower() == 'markdown': save_markdown(output_json) + elif mode.lower() == 'gource': + show_gource(output_json) else: print("[!] That output mode is not supported! Check README for help.") diff --git a/stats.py b/stats.py index fc96ce5..29b1db8 100644 --- a/stats.py +++ b/stats.py @@ -18,19 +18,24 @@ values['size'] = 'small' category = '' start = '' end = '' +logs = False weeks = 0 baseurl = "https://apps.fedoraproject.org/datagrepper/raw" -unicode_json={} +unicode_json = {} + def return_epoch(time): if time == '': return '' - tup = map(int,time.split('/')) + tup = map(int, time.split('/')) l = (tup[2], tup[0], tup[1], 0, 0, 0) epochs = calendar.timegm(l) return (int(epochs)) -# Checks if unicode_json is empty, pulls datagrepper values and returns the json +# Checks if unicode_json is empty, pulls datagrepper values and returns +# the json + + def return_json(): global unicode_json total_pages = 1 @@ -39,14 +44,16 @@ def return_json(): if len(unicode_json) == 0: print('[*] Grabbing datagrepper values..') - # If the user is set as all, we filter it using the provided category, if any + # If the user is set as all, we filter it using the provided category, + # if any if category != '' and values['user'] == 'all': values['category'] = category - if start != '' and end != '' : + if start != '' and end != '': values['start'] = return_epoch(start) values['end'] = return_epoch(end) del(values['delta']) - # If the user value is passed as all, remove it from the dict and pass arguments + # If the user value is passed as all, remove it from the dict and pass + # arguments if values['user'] == 'all': temp_dict = dict(values) del(temp_dict['user']) @@ -56,20 +63,22 @@ def return_json(): unicode_json = json.loads(response.text) total_pages = unicode_json['pages'] print ("Total pages found : " + str(total_pages)) - + total = total_pages # If multiple pages exist, get them all. while total_pages > 1: + print(" [*] Loading Page " + str(values['page']) + "/" + str(total)) values['page'] += 1 - print(" [*] Pulling data from page " + str(values['page'])) response = requests.get(baseurl, params=values) paginated_json = json.loads(response.text) # Pull data from multiple pages and append them to the main JSON - for activity in paginated_json['raw_messages'] : + for activity in paginated_json['raw_messages']: unicode_json['raw_messages'].append(activity) total_pages -= 1 return unicode_json # Analyzes the JSON and return categories present as a list. + + def return_categories(): cat_list = list() categories = Counter() @@ -82,7 +91,9 @@ def return_categories(): categories[category] += 1 return categories -# Given a category, looks for subcategories in the category and returns a sub-category counter. +# Given a category, looks for subcategories in the category and returns a +# sub-category counter. + def return_subcategories(category): subcat_list = list() subcategories = Counter() @@ -93,11 +104,13 @@ def return_subcategories(category): # Converts the list into a counter. for subcategory in subcat_list: - subcategories[subcategory] += 1 + subcategories[subcategory] += 1 return subcategories -# Gets the subcategories as a counter, analyzes it for further activities - named interactions. +# Gets the subcategories as a counter, analyzes it for further activities # Returns a counter with the found interactions + + def return_interactions(subcategories): interaction_dict = dict() interaction_list = list() @@ -110,11 +123,13 @@ def return_interactions(subcategories): for activity in unicode_json['raw_messages']: for object in subcategories: try: - if object == activity['topic'].split('.')[4] and activity['topic'].split('.')[5]: - interaction_dict[object].append(activity['topic'].split('.')[5]) + if object == activity['topic'].split('.')[4] and activity[ + 'topic'].split('.')[5]: + interaction_dict[object].append( + activity['topic'].split('.')[5]) except IndexError: print("[!] That category doesn't have any more interactions!") - return {None:None} + return {None: None} # Changing list to a counter for key in interaction_dict: diff --git a/us_pycon_autostats.py b/us_pycon_autostats.py index 39e5957..8fc1ef8 100644 --- a/us_pycon_autostats.py +++ b/us_pycon_autostats.py @@ -1,11 +1,15 @@ import os + def main(): - f = open('users.txt') - lines = [line.rstrip('\n') for line in open('users.txt')] - print lines + f = open('users.txt') + lines = [line.rstrip('\n') for line in open('users.txt')] + print lines - for user in lines: - os.system('python main.py -u ' + user + ' -s 05/28/2016 -e 06/05/2016 -m csv') + for user in lines: + os.system( + 'python main.py -u ' + + user + + ' -s 05/28/2016 -e 06/05/2016 -m csv') main() From 37bf5cef67f6919b47f3a12d703e7f7e8e03b5eb Mon Sep 17 00:00:00 2001 From: Sachin Kamath Date: Jun 21 2016 13:23:35 +0000 Subject: [PATCH 6/6] Added weekly stats --- diff --git a/weekly_intern_stats.py b/weekly_intern_stats.py new file mode 100644 index 0000000..8807e47 --- /dev/null +++ b/weekly_intern_stats.py @@ -0,0 +1,27 @@ +import os + + +def main(): + fp = open('interns.txt') + interns = [user.strip('\n') for user in open('interns.txt')] + print interns + + for intern in interns: + if not os.path.exists(intern): + os.makedirs(intern) + # CSV MAIN FILE + os.system('python main.py -u %s -s 05/23/2016 -e 06/21/2016 -m csv -o stats_main' %( + intern)) + # Category-wise and Pagure + os.system('python main.py -u %s -s 05/23/2016 -e 06/21/2016 -m svg -o %s/%s -c pagure' %( + intern, intern, intern)) + # Markdown Report + os.system('python main.py -u %s -s 05/23/2016 -e 06/21/2016 -m markdown -o %s/README.md' %( + intern, intern)) + # User Specific + if intern == 'dhanvi': + os.system('python main.py -u %s -s 05/23/2016 -e 06/21/2016 -m svg -o %s/%s -c copr' %( + intern, intern, intern)) + +if __name__ == '__main__': + main()