From dc521a2b2d795014327ef46c9e97d9820f66aadd Mon Sep 17 00:00:00 2001 From: Sergey Turivniy Date: Jan 03 2018 17:18:24 +0000 Subject: [PATCH 1/2] Add statistic page. Old one stable end new one --- diff --git a/statictic_page/new_page/.gitignore b/statictic_page/new_page/.gitignore new file mode 100644 index 0000000..931d5b2 --- /dev/null +++ b/statictic_page/new_page/.gitignore @@ -0,0 +1,3 @@ +.up* +*.pyc +.idea/ diff --git a/statictic_page/new_page/README.md b/statictic_page/new_page/README.md new file mode 100644 index 0000000..3d0ed66 --- /dev/null +++ b/statictic_page/new_page/README.md @@ -0,0 +1,46 @@ +#### Get packages statistic and generate file + +Navigate to folder with tests: +``` +$ cd AtomicCi/statictic_page/new_page/ +``` + +Create python virtualenv with python3 +and activate it: +``` +$ virtualenv -p python3 envname +$ . envname/bin/activate +``` + +Install requirenments: +``` +$ pip install -r requirenments.txt +``` + +Edit file config.py, add upstreamfirst_api_key and dist_git_api_key. You can generate then on settings page: +``` +upstreamfirst_api_key = '' +dist_git_api_key = '' + +UPSTREAMFIRST_URL = 'https://upstreamfirst.fedorainfracloud.org/' +DIST_GIT_URL = 'https://src.fedoraproject.org/' + +FILE_UPLOAD_2_WIKI = 'upload_to_wiki.txt' +FILE_PACKAGES = 'upstreamfirst_packages.txt' +``` +Also you can change names of the files FILE_UPLOAD_2_WIKI and FILE_PACKAGES + +To run code just execute in a command line: +``` +$ python runner.py +``` + +File upstreamfirst_packages.txt will be created. +You can simply check progress by executing following command in another terminal: +``` +$ tail -f upstreamfirst_packages.txt +``` + +After script will be finished, find upload_to_wiki.txt. Use data to upload it to the wiki. + +The result: https://fedoraproject.org/wiki/CI/Tests/new-stat#This_page_is_automatically_updated._Do_not_edit. \ No newline at end of file diff --git a/statictic_page/new_page/config.py b/statictic_page/new_page/config.py new file mode 100644 index 0000000..5262a6c --- /dev/null +++ b/statictic_page/new_page/config.py @@ -0,0 +1,8 @@ +upstreamfirst_api_key = '' +dist_git_api_key = '' + +UPSTREAMFIRST_URL = 'https://upstreamfirst.fedorainfracloud.org/' +DIST_GIT_URL = 'https://src.fedoraproject.org/' + +FILE_UPLOAD_2_WIKI = 'upload_to_wiki.txt' +FILE_PACKAGES = 'upstreamfirst_packages.txt' \ No newline at end of file diff --git a/statictic_page/new_page/generate_file.py b/statictic_page/new_page/generate_file.py new file mode 100644 index 0000000..2749247 --- /dev/null +++ b/statictic_page/new_page/generate_file.py @@ -0,0 +1,196 @@ +from tools import get_list_of_packages_from_the_file +from config import FILE_UPLOAD_2_WIKI +from config import FILE_PACKAGES + +def generate_file_upload_on_wiki(input_file): + """Generate file that will be uploaded on the wiki page + + Args: + input_file (file): file with packages + + Returns: + (file): file that will be uploaded on the wiki page + """ + + packages = get_list_of_packages_from_the_file(input_file) + packages_statistic = get_packages_statistic(packages) + + with open(FILE_UPLOAD_2_WIKI, 'w') as upload_file: + + upload_file.write('== This page is automatically updated. Do not edit. ==\n') + upload_file.write('{| class="wikitable sortable"\n') + upload_file.write('! style="text-align:left;" scope="col" rowspan="2" | Package\n') + upload_file.write('! scope="col" colspan="6"| dist-git\n') + upload_file.write('! scope="col" colspan="4"| upstreamfirst\n') + upload_file.write('|-\n') + upload_file.write('! scope="col" | tests.yml\n') + upload_file.write('! scope="col" | classic\n') + upload_file.write('! scope="col" | container\n') + upload_file.write('! scope="col" | atomic\n') + upload_file.write('! scope="col" | missing\n') + upload_file.write('! scope="col" | pending PR\n') + upload_file.write('! scope="col" | tests.yml\n') + upload_file.write('! scope="col" | classic\n') + upload_file.write('! scope="col" | container\n') + upload_file.write('! scope="col" | atomic\n') + upload_file.write('|-\n') + + # Statistics + upload_file.write('! style="text-align:left; font-weight: lighter;" scope="row" | Total known packages\n') + upload_file.write('| style="text-align:center;" scope="row" colspan="10" | {}\n'.format( + packages_statistic['total'])) + upload_file.write('|-\n') + upload_file.write('! style="text-align:left; font-weight: lighter;" scope="row" | Packages in category\n') + upload_file.write('| {}\n'.format(packages_statistic['dist-git']['test_yml'])) + upload_file.write('| {}\n'.format(packages_statistic['dist-git']['test_tags']['classic'])) + upload_file.write('| {}\n'.format(packages_statistic['dist-git']['test_tags']['container'])) + upload_file.write('| {}\n'.format(packages_statistic['dist-git']['test_tags']['atomic'])) + upload_file.write('| {}\n'.format(packages_statistic['dist-git']['missing'])) + upload_file.write('| {}\n'.format(packages_statistic['dist-git']['pending'])) + upload_file.write('| {}\n'.format(packages_statistic['upstreamfirst']['test_yml'])) + upload_file.write('| {}\n'.format(packages_statistic['upstreamfirst']['test_tags']['classic'])) + upload_file.write('| {}\n'.format(packages_statistic['upstreamfirst']['test_tags']['container'])) + upload_file.write('| {}\n'.format(packages_statistic['upstreamfirst']['test_tags']['atomic'])) + upload_file.write('|-\n') + + # fill up file with data + for package in packages: + dict_pack = eval(package) + + upload_file.write('!style="text-align:left;" scope="row" | {}\n'.format(dict_pack['name'])) + + upload_file.write(format_color(dict_pack) + format_test_yml(dict_pack['dist-git'])) # dist-git tests.yml + upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['dist-git']['test_tags']['classic'])) # dist-git classic + upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['dist-git']['test_tags']['container'])) # dist-git container + upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['dist-git']['test_tags']['atomic'])) # dist-git atomic + upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['dist-git']['missing'])) # dist-git missing + upload_file.write(format_color(dict_pack) + format_pending(dict_pack['dist-git']['pending'])) # dist-git pending + + upload_file.write(format_color(dict_pack) + format_test_yml(dict_pack['upstreamfirst'])) # upstreamfirst tests.yml + upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['upstreamfirst']['test_tags']['classic'])) # upstreamfirst classic + upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['upstreamfirst']['test_tags']['container'])) # upstreamfirst container + upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['upstreamfirst']['test_tags']['atomic'])) # upstreamfirst atomic + + upload_file.write('|-\n') + + upload_file.write('|} \n') + upload_file.close() + + +def format_pending(pending): + """Updates pending cell if package is on pending + + Args: + pending (dict): pending is dict_pack['dist-git']['pending'] + + Returns: + empty cell or cell with info + """ + if pending['status'] == '': + return '| \n' + else: + + user_str = pending['user']['name'] + ' ' + pending['user']['fullname'] + return '|[[{}|YES]] [{} _*] \n'.format(user_str, pending['url']) + + +def format_test_yml(test_yml): + """ + test_yml is dict_pack['upstreamfirst'] + or + test_yml is dict_pack['dist-git'] + """ + if test_yml['package_url']: + return '| [{} {}]\n'.format(test_yml['package_url'], test_yml['test_yml']) + else: + return '| \n' + + +def format_color(dict_pack): + """Function formats cell color to #7FFF00" 'Green' if package ready for porting""" + + if dict_pack['dist-git']['pending']['status'] == '' \ + and dict_pack['dist-git']['test_yml'] == '' \ + and dict_pack['upstreamfirst']['test_yml'] != '': + return '! style="background-color:#7FFF00"' + return '' + + +def get_packages_statistic(packages): + """Function gets packages statistic + + Args: + packages (list): list of packages + + Returns: + statistic_json (dict): packages statistic + """ + statistic_json = {'total': '', + 'dist-git': { + 'test_yml': '', + 'missing': '', + 'pending': '', + 'test_tags': {'classic': '', 'container': '', 'atomic': ''}}, + 'upstreamfirst': {'test_yml': '', + 'test_tags': {'classic': '', 'container': '', 'atomic': ''}}} + total_packages = len(packages) + statistic_json['total'] = str(total_packages) + statistic_json['dist-git']['test_yml'] = _get_packages_number_in_category(total_packages, packages, + 'dist-git', 'test_yml') + statistic_json['dist-git']['missing'] = _get_packages_number_in_category(total_packages, packages, + 'dist-git', 'missing') + statistic_json['dist-git']['pending'] = _get_packages_number_in_category(total_packages, packages, + 'dist-git', 'pending', 'status') + statistic_json['dist-git']['test_tags']['classic'] = _get_packages_number_in_category(total_packages, packages, + 'dist-git', + 'test_tags', 'classic') + statistic_json['dist-git']['test_tags']['container'] = _get_packages_number_in_category(total_packages, packages, + 'dist-git', + 'test_tags', 'container') + statistic_json['dist-git']['test_tags']['atomic'] = _get_packages_number_in_category(total_packages, packages, + 'dist-git', + 'test_tags', 'atomic') + + statistic_json['upstreamfirst']['test_yml'] = _get_packages_number_in_category(total_packages, packages, + 'upstreamfirst', 'test_yml') + statistic_json['upstreamfirst']['test_tags']['classic'] = _get_packages_number_in_category(total_packages, packages, + 'upstreamfirst', + 'test_tags', 'classic') + statistic_json['upstreamfirst']['test_tags']['container'] = _get_packages_number_in_category(total_packages, + packages, + 'upstreamfirst', + 'test_tags', + 'container') + statistic_json['upstreamfirst']['test_tags']['atomic'] = _get_packages_number_in_category(total_packages, packages, + 'upstreamfirst', + 'test_tags', 'atomic') + return statistic_json + + +def _get_packages_number_in_category(total_packages, all_packages, *args): + """ + Function returns updated statistic_json from the get_packages_statistic() + Args: + total_packages (int): total packages + all_packages (list of jsons): all packages + + Returns: + formatted_string (all_packages): + for example: '48 (42%)' + """ + number_in_category = 0 + for item in all_packages: + dict_pack = eval(item) + + if len(args) == 2: + if 'YES' in dict_pack[args[0]][args[1]]: + number_in_category += 1 + else: + if 'YES' in dict_pack[args[0]][args[1]][args[2]]: + number_in_category += 1 + formatted_string = "{} ({}%)".format(number_in_category, round((100*number_in_category)/total_packages)) + return formatted_string + + +if __name__ == '__main__': + exit(0) diff --git a/statictic_page/new_page/requirements.txt b/statictic_page/new_page/requirements.txt new file mode 100644 index 0000000..3866aaa --- /dev/null +++ b/statictic_page/new_page/requirements.txt @@ -0,0 +1,2 @@ +requests +mwclient \ No newline at end of file diff --git a/statictic_page/new_page/runner.py b/statictic_page/new_page/runner.py new file mode 100644 index 0000000..4396671 --- /dev/null +++ b/statictic_page/new_page/runner.py @@ -0,0 +1,80 @@ +from tools import * + +from generate_file import generate_file_upload_on_wiki + +dist_git_api_key = config.dist_git_api_key + +UPSTREAMFIRST_URL = config.UPSTREAMFIRST_URL +DIST_GIT_URL = config.DIST_GIT_URL + +package_json = {'name': '', + 'dist-git': { + 'package_url': '', + 'test_yml': '', + 'missing': '', + 'pending': {'status': '', 'url': '', 'user': {}}, + 'test_tags': {'classic': '', 'container': '', 'atomic': ''}}, + 'upstreamfirst': {'test_yml': '', + 'package_url': '', + 'test_tags': {'classic': '', 'container': '', 'atomic': ''}}} + + +def wrapper_for_get_pull_requests(package): + """Function updates package_json with correct information""" + + raw_text = get_pull_requests(DIST_GIT_URL, package, dist_git_api_key) + result = manage_pull_request(raw_text) + package_json['name'] = package + + if result is not None: + package_json['dist-git']['pending']['url'] = result['url'] + package_json['dist-git']['pending']['user'] = result['user'] + package_json['dist-git']['pending']['status'] = 'YES' + else: + package_json['dist-git']['pending']['url'] = '' + package_json['dist-git']['pending']['user'] = '' + package_json['dist-git']['pending']['status'] = '' + + # Get upstreamfirst test-tags + upstream_test_tags = handle_test_tags(UPSTREAMFIRST_URL, package) + upstream_test_tags = test_tags_to_dict(upstream_test_tags) + package_json['upstreamfirst']['test_tags'] = upstream_test_tags + + upstream_url_to_test_yml = get_url_to_test_yml(UPSTREAMFIRST_URL, package) + if check_if_test_yml_exists(upstream_url_to_test_yml): + package_json['upstreamfirst']['test_yml'] = 'YES' + package_json['upstreamfirst']['package_url'] = upstream_url_to_test_yml + else: + package_json['upstreamfirst']['test_yml'] = '' + package_json['upstreamfirst']['package_url'] = '' + + # Get dist-git test-tags + dist_git_test_tags = handle_test_tags(DIST_GIT_URL, package) + dist_git_test_tags = test_tags_to_dict(dist_git_test_tags) + package_json['dist-git']['test_tags'] = dist_git_test_tags + + dist_git_url_to_test_yml = get_url_to_test_yml(DIST_GIT_URL, package) + if check_if_test_yml_exists(dist_git_url_to_test_yml): + package_json['dist-git']['test_yml'] = 'YES' + package_json['dist-git']['package_url'] = dist_git_url_to_test_yml + else: + package_json['dist-git']['test_yml'] = '' + package_json['dist-git']['package_url'] = '' + + # Write results to the file + write_results_to_the_file(package_json, config.FILE_PACKAGES) + + +if __name__ == '__main__': + # Prepare environment + delete_files_before_running([config.FILE_PACKAGES, config.FILE_UPLOAD_2_WIKI]) + + # Get all open pull requests + projects = get_all_projects(UPSTREAMFIRST_URL, upstreamfirst_api_key) + all_packages = get_projects_names(projects) + + for package in all_packages: + wrapper_for_get_pull_requests(package) + + generate_file_upload_on_wiki(config.FILE_PACKAGES) + exit(0) diff --git a/statictic_page/new_page/tools.py b/statictic_page/new_page/tools.py new file mode 100644 index 0000000..379fdd1 --- /dev/null +++ b/statictic_page/new_page/tools.py @@ -0,0 +1,272 @@ +import os +import re +import requests + +import config + + +upstreamfirst_api_key = config.upstreamfirst_api_key +dist_git_api_key = config.dist_git_api_key + +UPSTREAMFIRST_URL = config.UPSTREAMFIRST_URL +DIST_GIT_URL = config.DIST_GIT_URL + + +def get_pull_requests(site_url, package, api_key): + """Get pull requests from site using API + + Args: + site_url (string): url of the site + for example: 'https://upstreamfirst.fedorainfracloud.org/' + package (string): name of the package + api_key (string): secret API key + + Returns: + response (json): info about PR + """ + site = site_url + 'api/0/rpms/' + package + '/pull-requests' + + response = requests.get(site, headers={'Authorization': 'access_token {}'.format(api_key)}) + return response.json() + + +def manage_pull_request(response_json): + """Checks if PR corresponds to the test PR and open. + + Args: + response_json (json): info about PR + + Returns: + (json): {'user': , 'url': } + """ + try: + if response_json['total_requests'] > 0: + for request in response_json['requests']: + if ('Add CI tests' or 'test' in request['title']) and (request['status'] == 'Open'): + pull_req_url = DIST_GIT_URL + request['project']['url_path'] + '/pull-requests' + return {'user': request['user'], 'url': pull_req_url} + + except Exception: + # {'error': 'Project not found', 'error_code': 'ENOPROJECT'} + return + + +def get_all_projects(site_url, api_key): + """Get all projects from site using API + + Args: + site_url (string): url of the site + for example: 'https://upstreamfirst.fedorainfracloud.org/' + api_key (string): secret API key + + Returns: + response (json): all projects + """ + site = site_url + 'api/0/projects' + + response = requests.get(site, headers={'Authorization': 'access_token {}'.format(api_key)}) + return response.json() + + +def get_projects_names(json_response): + """Get projects names from the all projects + + Args: + json_response (json): all projects + + Returns: + projects_names (list of strings) names of projects + """ + + projects_names = [] + + for project in json_response['projects']: + project_name = project['fullname'] + if 'forks/' not in project_name: + projects_names.append(project_name) + + return projects_names + + +def get_projects_url_patches(json_response): + """Get projects url patches""" + + projects_url_patches = [] + + for project in json_response['projects']: + project_name = project['url_path'] + + projects_url_patches.append(project_name) + + return projects_url_patches + + +def get_test_yaml_from_site(site_url, package, test_file='tests.yml'): + """Get test.yaml from the site + + Args: + site_url (string): url of the site + for example: 'https://upstreamfirst.fedorainfracloud.org/' + package (string): name of the package + test_file (string): file to get from site. + + Returns: + test.yaml (raw string) + """ + + if 'upstreamfirst' in site_url: + test_file_url = site_url + package + '/raw/master/f/' + test_file + elif 'fedoraproject' in site_url: + test_file_url = site_url + '/rpms/' + package + '/raw/master/f/tests/' + test_file + else: + return + + response = requests.get(test_file_url) + return response.text + + +def get_url_to_test_yml(site_url, package): + """Get url to the test.yml file + + Args: + site_url (string): url of the site + for example: 'https://upstreamfirst.fedorainfracloud.org/' + package (string): name of the package + + Returns: + url (string) + """ + if 'upstreamfirst' in site_url: + test_file_url = site_url + package + '/blob/master/f/tests.yml' + elif 'fedoraproject' in site_url: + test_file_url = site_url + 'rpms/' + package + '/blob/master/f/tests/tests.yml' + else: + return + + return test_file_url + + +def check_if_test_yml_exists(test_file_url): + """Checks if tesy.yml exists + + Args: + test_file_url (string): url of the site + + Returns: + True/False (bool) if response.status_code + """ + response = requests.get(test_file_url) + if response.status_code == 200: + return True + else: + return False + + +def get_test_tags(raw_text): + """Just returns existed test-tags + + Args: + raw_text (string): raw text output from the requests + + Returns: + tags (list if strings) + """ + test_tags = [] + + for tag in ['classic', 'container', 'atomic']: + if tag in raw_text: + test_tags.append(tag) + + return test_tags + + +def handle_test_tags(site_url, package): + """Function grts new patch to the test.yaml if existing test.yaml includes test file + + Args: + site_url (string): url of the site + for example: 'https://upstreamfirst.fedorainfracloud.org/' + package (string): name of the package + + Returns: + tags (list of strings) + """ + + raw_text = get_test_yaml_from_site(site_url, package) + + if 'Page not found' in raw_text: + return [] + + test_tags = get_test_tags(raw_text) + + if not test_tags: + new_test_file = re.findall(r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', raw_text) + + if new_test_file: + raw_text = get_test_yaml_from_site(site_url, package, new_test_file[-1]) + test_tags = get_test_tags(raw_text) + + return test_tags + + +def test_tags_to_dict(test_tags): + """Convert test-tags list to the dictionary""" + + tags_dict = {'classic': '', 'container': '', 'atomic': ''} + try: + for tag in test_tags: + tags_dict[tag] = 'YES' + + except Exception.KeyError: + pass + return tags_dict + + +def get_list_of_packages_from_the_file(file): + """Reads list of packages from a file + + Args: + file (file): file to get packages from + + Returns: + packages (list of packages) + """ + with open(file) as packages_file: + return packages_file.read().splitlines() + + +def write_results_to_the_file(result, filename='results.txt'): + """Write results to the file + + Args: + result (json): package's info + filename (string): name of the file to write packages + + Returns: + packages (list of packages) + """ + with open(filename, 'a') as my_file: + my_file.write('{}\n'.format(result)) + + my_file.close() + return filename + + +def delete_files_before_running(files_to_delete): + """Function deletes (cleanup) files with data + + Args: + files_to_delete (list): list of files to delete + + Returns: + (None) if everything is alright + """ + for file_name in files_to_delete: + try: + os.remove(file_name) + except OSError: + pass + + +if __name__ == '__main__': + exit(0) diff --git a/statictic_page/new_page/upload_to_wiki.py b/statictic_page/new_page/upload_to_wiki.py new file mode 100644 index 0000000..5c1d6e2 --- /dev/null +++ b/statictic_page/new_page/upload_to_wiki.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import mwclient +import os + +ua = 'MyWikiTool/0.2 run by User:FedoraUser' +site = mwclient.Site('fedoraproject.org', clients_useragent=ua) +# site.login(os.environ.get('WIKI_USER'), os.environ.get('WIKI_PASS')) +site.login('sturivny', 'VgaZsxPWrMnCzTWm') +page = site.pages['CI/Tests/new-stat'] + +page.exists + +with open('upload_file.txt', 'r') as myfile: + text = myfile.read() +page.save(text, 'Current status.') diff --git a/statictic_page/stable_page/koji-stats b/statictic_page/stable_page/koji-stats new file mode 100755 index 0000000..dd51f99 --- /dev/null +++ b/statictic_page/stable_page/koji-stats @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# +# Copyright Red Hat Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# This file is part of Cockpit. +# + +UPSTREAM_PAGURE = "https://upstreamfirst.fedorainfracloud.org/api/0/" +UPSTREAM_BASE = "https://upstreamfirst.fedorainfracloud.org/" + +DISTGIT_BASE = "https://src.fedoraproject.org/git/rpms/" + +import argparse +import json +import os +import random +import sys +import tempfile +import time +import shutil +import subprocess + +import urllib.request +import urllib.parse + +DISTGIT_BASE = "https://src.fedoraproject.org/git/rpms/" + +def main(): + parser = argparse.ArgumentParser(description="Gather koji stats") + parser.add_argument("--scratch", action="store", default="/tmp", help="Scratch directory") + opts = parser.parse_args() + + stats = { } + directory = tempfile.mkdtemp(prefix="koji-stats", dir=opts.scratch) + + # Dist git + repos = repositories() + + offset = random.randint(0, len(repos)) + while True: + offset = (offset + 1) % len(repos) + project = repos[offset] + + url = urllib.parse.urljoin(DISTGIT_BASE, project) + repo = clone(url, directory) + if not repo: + continue + + start = time.time() + local(repo) + duration = int(time.time() - start) + log("mock", project, str(duration)) + + start = time.time() + koji(repo) + duration = int(time.time() - start) + log("koji", project, str(duration)) + + start = time.time() + koji(repo, "x86_64") + duration = int(time.time() - start) + log("koji-x86_64", project, str(duration)) + + shutil.rmtree(repo) + + return 0 + +def repositories(): + results = [ ] + base = os.path.dirname(__file__) + with open(os.path.join(base, "repos"), "rb") as f: + while True: + line = f.readline() + if not line: + break + line = line.strip() + if line: + results.append(line.decode("utf-8")) + return results + +def log(self, *args): + line = "\t".join(args) + "\n" + sys.stdout.write(line) + with open(os.path.join(os.path.dirname(__file__), "koji-stats.log"), 'a') as f: + f.write(line) + +def clone(url, directory): + target = os.path.join(directory, os.path.basename(url)) + while os.path.exists(target): + target += ".x" + try: + subprocess.check_call(["git", "clone", url, target], cwd=directory, stdout=sys.stderr.fileno()) + except subprocess.CalledProcessError: + return None + subprocess.check_call(["git", "checkout", "f26"], cwd=target, stdout=sys.stderr.fileno()) + return target + +def koji(directory, arch=None): + if arch: + cmd = ["fedpkg", "--scratch", "--arches", arch, "build"] + else: + cmd = ["fedpkg", "--scratch", "build"] + subprocess.check_call(cmd, cwd=directory, stdout=sys.stderr.fileno()) + +def local(directory): + subprocess.check_call(["fedpkg", "mockbuild"], cwd=directory, stdout=sys.stderr.fileno()) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/statictic_page/stable_page/publish2wiki.py b/statictic_page/stable_page/publish2wiki.py new file mode 100644 index 0000000..86ac933 --- /dev/null +++ b/statictic_page/stable_page/publish2wiki.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +# Purpose: publish output of ./test-stats to Fedora Wiki. This script is +# scheduled on Centos OpenShift: apps.ci.centos.org. Dependency are installed +# as a part of Job definition. data.txt comes from ./test-stats output. + +import mwclient +import os + +ua = 'MyWikiTool/0.2 run by User:FedoraUser' +site = mwclient.Site('fedoraproject.org', clients_useragent=ua) +site.login(os.environ.get('WIKI_USER'), os.environ.get('WIKI_PASS')) +page = site.pages['CI/Tests/stat'] +page.exists +with open('data.txt', 'r') as myfile: + text = myfile.read() +page.save(text, 'Current status.') diff --git a/statictic_page/stable_page/repos b/statictic_page/stable_page/repos new file mode 100644 index 0000000..b21aa98 --- /dev/null +++ b/statictic_page/stable_page/repos @@ -0,0 +1,288 @@ +acl +atomic +atomic-devmode +attr +audit +authconfig +avahi +basesystem +bash +bash-completion +bind +binutils +biosdevname +boost +bridge-utils +bzip2 +ca-certificates +c-ares +ceph +checkpolicy +chkconfig +chrony +cloud-init +cloud-utils +cockpit +conntrack-tools +container-selinux +coreutils +cpio +cracklib +criu +cronie +crontabs +cryptsetup +cups +curl +cyrus-sasl +dbus +dbus-glib +dbus-python +device-mapper +device-mapper-multipath +device-mapper-persistent-data +dhcp +diffutils +dmidecode +dnf +dnsmasq +docker +docker-client +docker-latest +dracut +e2fsprogs +efibootmgr +efivar +etcd +ethtool +expat +fcgi +file +filesystem +findutils +fipscheck +flannel +freetype +fuse +gawk +gdbm +GeoIP +gettext +glib2 +glibc +glib-networking +glusterfs +gmp +gnupg2 +gnutls +gobject-introspection +gomtree +gperftools +gpgme +grep +groff +grub2 +grubby +gsettings-desktop-schemas +gssproxy +gzip +hardlink +hostname +initscripts +iproute +iptables +iputils +irqbalance +iscsi-initiator-utils +jansson +json-glib +kernel +kexec-tools +keyutils +kmod +krb5 +kubernetes +less +libaio +libarchive +libassuan +libcap +libcap-ng +libcgroup +libcollection +libcroco +libdb +libdhash +libedit +libevent +libffi +gcc +libgcrypt +libgpg-error +icu +libidn +libini_config +ipa +libldb +libmnl +libmodman +libndp +libnetfilter_conntrack +libnetfilter_cthelper +libnetfilter_cttimeout +libnetfilter_queue +libnfnetlink +libnfsidmap +libnl +libnl3 +libpath_utils +libpcap +libproxy +libpwquality +libref_array +librepo +libreport +libseccomp +libselinux +libsemanage +libsepol +libsolv +libsoup +libssh2 +sssd +libtalloc +libtasn1 +libtdb +libtevent +libtirpc +libunistring +libunwind +libuser +libutempter +uuid +libverto +libxml2 +libyaml +linux-firmware +logrotate +lttng-ust +lua +lvm2 +lzo +m2crypto +make +mdadm +mokutil +mozjs17 +nano +ncurses +nettle +net-tools +NetworkManager +newt +nfs-utils +nmap +nspr +nss +nss-altfiles +nss-softokn +nss-util +oci-register-machine +oci-systemd-hook +openldap +openssh +openssl +os-prober +ostree +p11-kit +pam +passwd +pciutils +pcre +pinentry +pkgconfig +policycoreutils +polkit +polkit-pkla-compat +popt +ppp +procps-ng +protobuf-c +pth +pygpgme +pyliblzma +python +python-backports +python-backports-ssl_match_hostname +python-chardet +python-configobj +python-dateutil +python-decorator +python-dmidecode +python-docker-py +python-docker-pycreds +python-ethtool +python-iniparse +python-ipaddress +python-IPy +python-jsonpatch +python-jsonpointer +python-prettytable +python-pycurl +python-pyudev +python-requests +python-rhsm +python-setuptools +python-six +python-slip +python-urlgrabber +python-urllib3 +python-websocket-client +pyxattr +PyYAML +quota +readline +rpcbind +rpm +rpm-ostree +rsync +runc +samba +sed +selinux-policy +setools +setup +sg3_utils +shadow-utils +shared-mime-info +shim +skopeo +slang +snappy +socat +sqlite +sssd +subscription-manager +sudo +systemd +sysvinit +tar +tcp_wrappers +tmux +trousers +tuned +tzdata +usermode +userspace-rcu +ustr +util-linux +vim +virt-what +which +wpa_supplicant +xfsprogs +xz +yajl +yum +yum-metadata-parser +zlib diff --git a/statictic_page/stable_page/requirements.txt b/statictic_page/stable_page/requirements.txt new file mode 100644 index 0000000..df27baa --- /dev/null +++ b/statictic_page/stable_page/requirements.txt @@ -0,0 +1 @@ +mwclient>=0.8.6 \ No newline at end of file diff --git a/statictic_page/stable_page/run-tests b/statictic_page/stable_page/run-tests new file mode 100755 index 0000000..dc21486 --- /dev/null +++ b/statictic_page/stable_page/run-tests @@ -0,0 +1,20 @@ +#!/bin/sh -euf + +# Prepare atomic host to test, report and artifacts dirs +export TEST_SUBJECTS="$PWD/atomic.qcow2" +test -f $TEST_SUBJECTS || curl -o $TEST_SUBJECTS http://artifacts.ci.centos.org/fedora-atomic/f26/images/latest-atomic.qcow2 +rm -rf report.txt artifacts/ clone/ + +# Go through each project in upstreamfirst, clone it and put pass/fail in report.txt +curl https://upstreamfirst.fedorainfracloud.org/api/0/projects?short=true | grep '"fullname"' | grep -v '/' | cut -d '"' -f 4 | while read repo; do + git clone https://upstreamfirst.fedorainfracloud.org/$repo clone/$repo + export ANSIBLE_INVENTORY=$(test -e $repo/inventory && echo clone/$repo/inventory || echo /usr/share/ansible/inventory) + export TEST_ARTIFACTS="$PWD/artifacts/$repo" + if ! ansible-playbook -i localhost, --list-tags clone/$repo/tests.yml | grep atomic; then + printf "$repo skip\n" >> report.txt + elif ansible-playbook -t atomic -e "subjects=$TEST_SUBJECTS" -e "artifacts=$TEST_ARTIFACTS" clone/$repo/tests.yml; then + printf "$repo pass\n" >> report.txt + else + printf "$repo fail\n" >> report.txt + fi +done diff --git a/statictic_page/stable_page/run-tests-parallel b/statictic_page/stable_page/run-tests-parallel new file mode 100755 index 0000000..ccb7de5 --- /dev/null +++ b/statictic_page/stable_page/run-tests-parallel @@ -0,0 +1,27 @@ +#!/bin/sh -euf + +# Prepare atomic host to test, report and artifacts dirs +export TEST_SUBJECTS="$PWD/atomic.qcow2" +test -f $TEST_SUBJECTS || curl -o $TEST_SUBJECTS http://artifacts.ci.centos.org/fedora-atomic/f26/images/latest-atomic.qcow2 +rm -rf repo/ report/ +mkdir report/ repo/ + +run() { + git clone https://upstreamfirst.fedorainfracloud.org/$1 repo/$1 + inventory=$(test -e $1/inventory && echo repo/$1/inventory || echo /usr/share/ansible/inventory) + export TEST_ARTIFACTS="$PWD/repo/artifacts/$1" + if ! ansible-playbook --list-tags repo/$1/tests.yml | grep atomic; then + printf "$1 skip\n" >> report/$1.txt + elif ansible-playbook -i $inventory -t atomic -e "subjects=$TEST_SUBJECTS" -e "artifacts=$TEST_ARTIFACTS" repo/$1/tests.yml; then + printf "$1 pass\n" >> report/$1.txt + else + printf "$1 fail\n" >> report/$1.txt + fi +} + +# Go through each project in upstreamfirst, clone it and put pass/fail in report.txt +curl https://upstreamfirst.fedorainfracloud.org/api/0/projects?short=true | grep '"fullname"' | grep -v '/' | cut -d '"' -f 4 | while read repo; do + sleep 3 + if test $(jobs | wc -l) -gt 16; then wait; fi + run $repo & +done diff --git a/statictic_page/stable_page/test-stats b/statictic_page/stable_page/test-stats new file mode 100755 index 0000000..8e70d1e --- /dev/null +++ b/statictic_page/stable_page/test-stats @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright Red Hat Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# This file is part of Cockpit. +# + +import argparse +import json +import os +import sys +import tempfile +import shutil +import subprocess + +import urllib.request +import urllib.parse + + +UPSTREAM_PAGURE = "https://upstreamfirst.fedorainfracloud.org/api/0/" +UPSTREAM_BASE = "https://upstreamfirst.fedorainfracloud.org/" + +DISTGIT_BASE = "https://src.fedoraproject.org/git/rpms/" + + +def retrieve(url): + return urllib.request.urlopen(url).read() + + +def projects(): + url = urllib.parse.urljoin(UPSTREAM_PAGURE, "projects") + for project in json.loads(retrieve(url))["projects"]: + if not project.get("parent"): + yield project["name"] + + +def repos(): + base = os.path.dirname(__file__) + with open(os.path.join(base, "repos"), "rb") as f: + while True: + line = f.readline() + if not line: + break + line = line.strip() + if line: + yield line.decode("utf-8") + + +def clone(url, directory): + target = os.path.join(directory, os.path.basename(url)) + while os.path.exists(target): + target += ".x" + try: + subprocess.check_call(["git", "clone", url, target], cwd=directory, stdout=sys.stderr.fileno()) + except subprocess.CalledProcessError: + return None + return target + + +def count(stats, field, project): + if field not in stats: + stats[field] = [ ] + stats[field].append(project) + + +def tags(repo, standard): + try: + playbook = standard and "tests.yml" or "test_local.yml" + environ = os.environ.copy() + environ["LC_ALL"] = "C" + output = subprocess.check_output([ "ansible-playbook", "--list-tags", playbook ], env=environ, cwd=repo) + except subprocess.CalledProcessError: + output = b"" + + for line in output.decode("utf-8").split("\n"): + if "TASK TAGS" in line: + name, unused, tags = line.partition(":") + for tag in tags.strip(" []").split(","): + yield tag.strip() + + +def gather(repo, project, stats): + + # Standard entry point + path = os.path.join(repo, "tests.yml") + standard = os.path.exists(path) + if standard: + count(stats, "tests.yml", project) + + # Check if we have an old style test_cloud.yml file + path = os.path.join(repo, "test_cloud.yml") + if os.path.exists(path): + count(stats, "atomic", project) + elif "atomic" in tags(repo, standard): + count(stats, "atomic", project) + + if os.path.exists(os.path.join(repo, "test_cloud.yml")) or \ + os.path.exists(os.path.join(repo, "test_local.yml")) or \ + os.path.exists(os.path.join(repo, "test_docker.yml")): + count(stats, "legacy", project) + + +def main(): + parser = argparse.ArgumentParser(description='Gather stats about tests in dist-git') + opts = parser.parse_args() + + stats = {} + directory = tempfile.mkdtemp(prefix="test-stats") + + # Dist git + for project in repos(): + url = urllib.parse.urljoin(DISTGIT_BASE, project) + repo = clone(url, directory) + if repo: + sub = os.path.join(repo, "tests") + count(stats, "distgit", project) + if os.path.exists(sub): + count(stats, "distgit-tests", project) + gather(sub, project, stats) + else: + count(stats, "missing", project) + + # Upstream first projects + for project in projects(): + url = urllib.parse.urljoin(UPSTREAM_BASE, project) + repo = clone(url, directory) + if repo: + count(stats, "upstream", project) + gather(repo, project, stats) + + shutil.rmtree(directory) + + for name in list(stats.keys()): + stats["{0}-length".format(name)] = len(stats[name]) + json.dump(stats, sys.stdout, indent=4) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From 9d2f476c30e86435f21b4100a5912887074a316e Mon Sep 17 00:00:00 2001 From: Sergey Turivniy Date: Jan 10 2018 17:56:59 +0000 Subject: [PATCH 2/2] Add functionality to get packages from URL --- diff --git a/statictic_page/stable_page/.gitignore b/statictic_page/stable_page/.gitignore new file mode 100644 index 0000000..bcf1aa4 --- /dev/null +++ b/statictic_page/stable_page/.gitignore @@ -0,0 +1,2 @@ +*.txt +.idea* diff --git a/statictic_page/stable_page/bash-job.yml b/statictic_page/stable_page/bash-job.yml new file mode 100644 index 0000000..7575532 --- /dev/null +++ b/statictic_page/stable_page/bash-job.yml @@ -0,0 +1,41 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: tests2wiki +spec: + parallelism: 1 + completions: 1 + template: + metadata: + name: pi + spec: + containers: + - name: pi + image: python + command: + - bash + - -c + - | + cd /tmp + HOME=/tmp pip install --user virtualenv + PYTHONPATH="/tmp/.local/lib/python3.6/site-packages:$PYTHONPATH" /tmp/.local/bin/virtualenv /tmp/v + source /tmp/v/bin/activate + pip install -r /tmp/fedora-ci-stats/requirements.txt + export GIT_COMMITTER_NAME=git + export GIT_COMMITTER_EMAIL=git@git.git + git clone --branch wikireport https://github.com/Andrei-Stepanov/fedora-ci-stats + git clone https://pagure.io/standard-test-roles.git + echo -e '[defaults]\nroles_path=/tmp/standard-test-roles/roles' > /tmp/ansible.cfg + export ANSIBLE_CONFIG=/tmp/ansible.cfg + pushd fedora-ci-stats + python test-stats.py --wikiout wdata.mw --reposlist repos + python test-stats.py --wikiout wdata_hp.mw --reposlist repos_hp + cat wdata.mw + cat wdata_hp.mw + #export WIKI_USER= + #export WIKI_PASS= + ./publish2wiki.py --filedoc wdata.mw --pagepath CI/Tests/stat + ./publish2wiki.py --filedoc wdata_hp.mw --pagepath CI/Tests/stat_hp + restartPolicy: OnFailure + +# vim: et ts=2 sw=2 ai diff --git a/statictic_page/stable_page/config.py b/statictic_page/stable_page/config.py new file mode 100644 index 0000000..bbc43c1 --- /dev/null +++ b/statictic_page/stable_page/config.py @@ -0,0 +1,12 @@ +UPSTREAMFIRST_URL = 'https://upstreamfirst.fedorainfracloud.org/' +DIST_GIT_URL = 'https://src.fedoraproject.org/' + +FILE_UPLOAD_2_WIKI = 'upl2wiki.txt' +FILE_PACKAGES = 'upstreamfirst_packages.txt' + +DEFAULT_URL='fedoraproject.org' +UPSTREAM_PAGURE = "https://upstreamfirst.fedorainfracloud.org/api/0/" +UPSTREAM_BASE = "https://upstreamfirst.fedorainfracloud.org/" +DISTGIT_BASE = "https://src.fedoraproject.org/git/rpms/" + +IN_J2_WIKI_TEMPLATE = 'stat.j2' \ No newline at end of file diff --git a/statictic_page/stable_page/dump2wiki-cronjob.yml b/statictic_page/stable_page/dump2wiki-cronjob.yml new file mode 100644 index 0000000..4e8ca51 --- /dev/null +++ b/statictic_page/stable_page/dump2wiki-cronjob.yml @@ -0,0 +1,45 @@ +apiVersion: batch/v2alpha1 +kind: CronJob +metadata: + name: tests2wiki +spec: + schedule: "11 9,18 * * *" + ConcurrencyPolicy: Replace + jobTemplate: + spec: + template: + metadata: + labels: + parent: "cronjobpi" + spec: + containers: + - name: wiki + image: python + command: + - bash + - -c + - | + cd /tmp + HOME=/tmp pip install --user virtualenv + PYTHONPATH="/tmp/.local/lib/python3.6/site-packages:$PYTHONPATH" /tmp/.local/bin/virtualenv /tmp/v + source /tmp/v/bin/activate + pip install -r /tmp/fedora-ci-stats/requirements.txt + export GIT_COMMITTER_NAME=git + export GIT_COMMITTER_EMAIL=git@git.git + git clone --branch wikireport https://github.com/Andrei-Stepanov/fedora-ci-stats + git clone https://pagure.io/standard-test-roles.git + echo -e '[defaults]\nroles_path=/tmp/standard-test-roles/roles' > /tmp/ansible.cfg + export ANSIBLE_CONFIG=/tmp/ansible.cfg + pushd fedora-ci-stats + python test-stats.py --wikiout wdata.mw --reposlist repos + python test-stats.py --wikiout wdata_hp.mw --reposlist repos_hp + cat wdata.mw + cat wdata_hp.mw + #export WIKI_USER= + #export WIKI_PASS= + ./publish2wiki.py --filedoc wdata.mw --pagepath CI/Tests/stat + ./publish2wiki.py --filedoc wdata_hp.mw --pagepath CI/Tests/stat_hp + restartPolicy: Never + + +# vim: et ts=2 sw=2 ai diff --git a/statictic_page/stable_page/generate_file_upload_wiki.py b/statictic_page/stable_page/generate_file_upload_wiki.py new file mode 100644 index 0000000..cfdc687 --- /dev/null +++ b/statictic_page/stable_page/generate_file_upload_wiki.py @@ -0,0 +1,124 @@ +import argparse +import datetime +import jinja2 +import os + +from tools import get_list_of_packages_from_the_file +from config import IN_J2_WIKI_TEMPLATE + + +def get_list_of_dicts(dict_of_strings): + result = [] + for package in dict_of_strings: + result.append(eval(package)) + return result + + +def generate_file_upload_on_wiki(input_file, output_file): + """Generate file that will be uploaded on the wiki page + + Args: + input_file (file): file with packages + output_file (file): file with packages in json format + + Returns: + (file): file that will be uploaded on the wiki page + """ + THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + packages = get_list_of_packages_from_the_file(input_file) + packages_statistic = get_packages_statistic(packages) + packages = get_list_of_dicts(packages) + + j2_loader = jinja2.FileSystemLoader(THIS_DIR) + j2_env = jinja2.Environment(loader=j2_loader, trim_blocks=True) + template = j2_env.get_template(IN_J2_WIKI_TEMPLATE) + + template_vars = {} + template_vars['updated'] = datetime.datetime.utcnow() + template_vars['total'] = packages_statistic + template_vars['pkgs'] = packages + + with open(output_file, 'w') as mwfile: + + for line in template.render(template_vars): + mwfile.write(line) + + +def get_packages_statistic(packages): + """Function gets packages statistic + + Args: + packages (list): list of packages + + Returns: + statistic_json (dict): packages statistic + """ + statistic_json = {'total': '', + 'distgit': { + 'test_yml': '', + 'missing': '', + 'pending': '', + 'test_tags': {'classic': '', 'container': '', 'atomic': ''}}, + 'upstreamfirst': {'test_yml': '', + 'test_tags': {'classic': '', 'container': '', 'atomic': ''}}} + total_packages = len(packages) + statistic_json['total'] = str(total_packages) + statistic_json['distgit']['test_yml'] = _get_packages_number_in_category(total_packages, packages, + 'distgit', 'test_yml') + statistic_json['distgit']['missing'] = _get_packages_number_in_category(total_packages, packages, + 'distgit', 'missing') + statistic_json['distgit']['pending'] = _get_packages_number_in_category(total_packages, packages, + 'distgit', 'pending', 'status') + statistic_json['distgit']['test_tags']['classic'] = _get_packages_number_in_category(total_packages, packages, + 'distgit', + 'test_tags', 'classic') + statistic_json['distgit']['test_tags']['container'] = _get_packages_number_in_category(total_packages, packages, + 'distgit', + 'test_tags', 'container') + statistic_json['distgit']['test_tags']['atomic'] = _get_packages_number_in_category(total_packages, packages, + 'distgit', + 'test_tags', 'atomic') + + statistic_json['upstreamfirst']['test_yml'] = _get_packages_number_in_category(total_packages, packages, + 'upstreamfirst', 'test_yml') + statistic_json['upstreamfirst']['test_tags']['classic'] = _get_packages_number_in_category(total_packages, packages, + 'upstreamfirst', + 'test_tags', 'classic') + statistic_json['upstreamfirst']['test_tags']['container'] = _get_packages_number_in_category(total_packages, + packages, + 'upstreamfirst', + 'test_tags', + 'container') + statistic_json['upstreamfirst']['test_tags']['atomic'] = _get_packages_number_in_category(total_packages, packages, + 'upstreamfirst', + 'test_tags', 'atomic') + return statistic_json + + +def _get_packages_number_in_category(total_packages, all_packages, *args): + """ + Function returns updated statistic_json from the get_packages_statistic() + Args: + total_packages (int): total packages + all_packages (list of jsons): all packages + + Returns: + formatted_string (all_packages): + for example: '48 (42%)' + """ + number_in_category = 0 + for item in all_packages: + dict_pack = eval(item) + + if len(args) == 2: + if dict_pack[args[0]][args[1]]: + number_in_category += 1 + else: + if dict_pack[args[0]][args[1]][args[2]]: + number_in_category += 1 + formatted_string = "{} ({}%)".format(number_in_category, round((100*number_in_category)/total_packages)) + return formatted_string + + +if __name__ == '__main__': + exit(0) diff --git a/statictic_page/stable_page/publish2wiki.py b/statictic_page/stable_page/publish2wiki.py old mode 100644 new mode 100755 index 86ac933..ba33973 --- a/statictic_page/stable_page/publish2wiki.py +++ b/statictic_page/stable_page/publish2wiki.py @@ -1,17 +1,78 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- -# Purpose: publish output of ./test-stats to Fedora Wiki. This script is -# scheduled on Centos OpenShift: apps.ci.centos.org. Dependency are installed -# as a part of Job definition. data.txt comes from ./test-stats output. +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2017 +# Author: Andrei Stepanov +# + +"""Purpose: Publish some document on MediaWiki. + +""" -import mwclient import os +import logging +import mwclient +import argparse + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +parser = argparse.ArgumentParser( + description='Publish a document on MediaWiki.') + + +DEFAULT_URL='fedoraproject.org' + +parser.add_argument("--url", metavar='URL', help="Wiki URL. Default: %s" + % DEFAULT_URL, default=DEFAULT_URL) + +parser.add_argument("--login", metavar='LOGIN', + help=("Wiki user. " + "If not set takes from Env variable: $WIKI_LOGIN")) + +parser.add_argument("--passw", metavar='PASSWORD', + help=("Wiki user password. " + "If not set takes from Env variable: $WIKI_PASS")) + +parser.add_argument("--filedoc", metavar='DOCUMENT', + required=True, help="Path to a document file to publish.") + +parser.add_argument("--pagepath", metavar='PAGEPATH', required=True, + help=("Path to Wiki page to be updated. " + "Example: Section/Sub/Doc")) + +args = parser.parse_args() + +login = args.login +if not login: + login = os.environ.get('WIKI_USER') + +passw = args.passw +if not passw: + passw = os.environ.get('WIKI_PASS') + +logger.info("URL: %s", args.url) +logger.info("LOGIN: %s", login) +logger.info("DOCUMENT FILE: %s", args.filedoc) +logger.info("WIKI PAGE: %s", args.pagepath) ua = 'MyWikiTool/0.2 run by User:FedoraUser' -site = mwclient.Site('fedoraproject.org', clients_useragent=ua) -site.login(os.environ.get('WIKI_USER'), os.environ.get('WIKI_PASS')) -page = site.pages['CI/Tests/stat'] -page.exists -with open('data.txt', 'r') as myfile: - text = myfile.read() -page.save(text, 'Current status.') +site = mwclient.Site(args.url, clients_useragent=ua) +site.login(login, passw) +page = site.pages[args.pagepath] +if not page.exists: + logger.info("Page %s doesn't exist. Create a new one.", args.pagepath) +with open(args.filedoc, 'r') as doc: + text = doc.read() +page.save(text, 'Auto updated.') diff --git a/statictic_page/stable_page/repos b/statictic_page/stable_page/repos index b21aa98..270acd3 100644 --- a/statictic_page/stable_page/repos +++ b/statictic_page/stable_page/repos @@ -286,3 +286,55 @@ yajl yum yum-metadata-parser zlib +# H&P components. Contact: Rachel Sibley +babeltrace +byacc +cflow +crypto-policies +ctags +dwz +elfutils +emacs +fedora-logos +fedora-release +fedora-repos +fedpkg-minimal +flex +fpc-srpm-macros +ghc-srpm-macros +gnat-srpm-macros +go-srpm-macros +guile +hmaccalc +isl +ksh +libatomic_ops +libipt +libksba +libmetalink +libmpc +libpipeline +libpng +libpsl +libsigsegv +libtool +lz4 +m4 +man-db +mpfr +ncompress +nghttp2 +npth +nss-pem +ocaml-srpm-macros +perl-srpm-macros +psmisc +publicsuffix-list +python3 +python-rpm-macros +qrencode +qt5 +redhat-rpm-config +sharutils +texinfo +unzip diff --git a/statictic_page/stable_page/repos_hp b/statictic_page/stable_page/repos_hp new file mode 100644 index 0000000..a9b7fdc --- /dev/null +++ b/statictic_page/stable_page/repos_hp @@ -0,0 +1,322 @@ +acl +alsa-lib +attr +audit +augeas +b43-openfwwf +babeltrace +basesystem +bash +bc +bind99 +binutils +biosdevname +bluez +btrfs-progs +byacc +bzip2 +ca-certificates +c-ares +ceph +checkpolicy +checksec +chkconfig +chrpath +cmake +cockpit +compat-openssl10 +coreutils +cpio +cracklib +crypto-policies +cryptsetup +curl +cyrus-sasl +dbus +dbus-glib +dbxtool +deltarpm +desktop-file-utils +device-mapper-multipath +dhcp +diffutils +ding-libs +dmraid +dnf +docbook-dtds +docbook-style-xsl +dosfstools +dracut +dtc +dwz +e2fsprogs +ebtables +efibootmgr +efivar +elfutils +emacs +ethtool +expat +fbset +fcoe-utils +fedora-logos +fedora-modular-release +fedora-modular-repos +fedora-repos +fedpkg-minimal +file +filesystem +findutils +fipscheck +flex +fpc-srpm-macros +freetype +fuse +gawk +gc +gcc +gdb +gdbm +gdisk +gettext +ghc-srpm-macros +glib2 +glibc +glibc32 +glib-networking +gmp +gnat-srpm-macros +gnupg +gnupg2 +gnutls +gobject-introspection +go-srpm-macros +gpart +gpgme +grep +groff +grub2 +grubby +gsettings-desktop-schemas +gssproxy +guile +gzip +hfsplus-tools +hfsutils +hmaccalc +hostname +hwdata +icu +ima-evm-utils +initscripts +ipcalc +iproute +ipset +iptables +iputils +iscsi-initiator-utils +isl +isns-utils +jansson +jbigkit +json-c +json-glib +kbd +kernel +kexec-tools +keyutils +kmod +krb5 +less +libaio +libarchive +libassuan +libatomic_ops +libblockdev +libcap +libcap-ng +libcgroup +libcomps +libconfig +libcroco +libdaemon +libdb +libdnf +libedit +libev +libevent +libffi +libgcrypt +libgpg-error +libidn +libidn2 +libipt +libjpeg-turbo +libksba +libmetalink +libmnl +libmodman +libmpc +libnetfilter_conntrack +libnfnetlink +libnfsidmap +libnl3 +libpcap +libpciaccess +libpeas +libpipeline +libpng +libproxy +libpwquality +librepo +libreport +librtas +libseccomp +libsecret +libselinux +libsemanage +libsepol +libservicelog +libsigsegv +libsolv +libssh +libssh2 +libtalloc +libtasn1 +libtevent +libtiff +libtirpc +libtool +libunistring +libunwind +libusb +libusbx +libuser +libutempter +libverto +libvpd +libxml2 +libxslt +libyaml +linux-firmware +lksctp-tools +lldpad +logrotate +lsof +lsscsi +lsvpd +lttng-ust +lua +lvm2 +lz4 +lzo +m4 +mactel-boot +make +man-db +mcstrans +mdadm +memtest86+ +microdnf +modulemd +mokutil +mpfr +mtools +multilib-rpm-config +nano +ncurses +nettle +nfs-utils +nghttp2 +npth +nspr +nss +nss-pem +nss-softokn +nss-util +ocaml-srpm-macros +openblas-srpm-macros +openjpeg2 +openldap +openssh +openssl +os-prober +p11-kit +pam +parted +passwd +patch +pciutils +pcre +pcre2 +perl-srpm-macros +pkgconf +plymouth +policycoreutils +polkit +popt +powerpc-utils +ppc64-diag +ppc64-utils +procps-ng +psmisc +publicsuffix-list +python3 +python-dateutil +python-iniparse +python-IPy +python-pip +python-rpm-macros +python-setuptools +python-six +python-smartcols +PyYAML +qrencode +qt5 +quota +readline +redhat-rpm-config +rootfiles +rpcbind +rpm +rsync +rust-srpm-macros +s390utils +sed +selinux-policy +servicelog +setools +setup +sg3_utils +sgml-common +sgpio +shadow-utils +shared-mime-info +shim-signed +snappy +sqlite +sudo +syslinux +systemd +tar +tcl +tcp_wrappers +texinfo +time +tmux +tzdata +uboot-tools +unzip +usermode +userspace-rcu +util-linux +vim +volume_key +which +words +xfsprogs +xmlrpc-c +xz +zip +zlib +zstd diff --git a/statictic_page/stable_page/requirements.txt b/statictic_page/stable_page/requirements.txt index df27baa..1587356 100644 --- a/statictic_page/stable_page/requirements.txt +++ b/statictic_page/stable_page/requirements.txt @@ -1 +1,5 @@ -mwclient>=0.8.6 \ No newline at end of file +requests +mwclient +jinja2>=2.10 +ansible +argparse \ No newline at end of file diff --git a/statictic_page/stable_page/stat.j2 b/statictic_page/stable_page/stat.j2 new file mode 100644 index 0000000..0768d5c --- /dev/null +++ b/statictic_page/stable_page/stat.j2 @@ -0,0 +1,73 @@ +== This page is automatically updated. Do not edit. == + +Page was updated on: {{updated}} UTC + +{% raw %} +{| class="wikitable sortable" +{% endraw %} +! style="text-align:left;" scope="col" rowspan="2" | Package +! scope="col" colspan="6"| dist-git +! scope="col" colspan="4"| upstreamfirst +|- +! scope="col" | tests.yml +! scope="col" | classic +! scope="col" | container +! scope="col" | atomic +! scope="col" | missing +! scope="col" | pending PR +! scope="col" | tests.yml +! scope="col" | classic +! scope="col" | container +! scope="col" | atomic +|- +! style="text-align:left; font-weight: lighter;" scope="row" | Total known packages +| style="text-align:center;" scope="row" colspan="10" | {{total.total}} +|- +! style="text-align:left; font-weight: lighter;" scope="row" | Packages in category +|{{total.distgit.test_yml}} +|{{total.distgit.test_tags.classic}} +|{{total.distgit.test_tags.atomic}} +|{{total.distgit.test_tags.container}} +|{{total.distgit.missing}} +|{{total.distgit.pending}} +|{{total.upstreamfirst.test_yml}} +|{{total.upstreamfirst.test_tags.classic}} +|{{total.upstreamfirst.test_tags.container}} +|{{total.upstreamfirst.test_tags.atomic}} +{% for package in pkgs %} +|- +! style="text-align:left;" scope="row" | {{package.name}} +|{% if package.distgit.test_yml %} yes {% else %} - {% endif %} [https://src.fedoraproject.org/rpms/{{package.name}}/tree/master *] +|{% if package.distgit.test_tags.classic %} yes {% else %} - {% endif %} + +|{% if package.distgit.test_tags.container %} yes {% else %} - {% endif %} + +|{% if package.distgit.test_tags.atomic %} yes {% else %} - {% endif %} + +|{% if package.distgit.missing %} yes {% else %} - {% endif %} + +|{% if package.distgit.pending.status %} yes [https://src.fedoraproject.org/rpms/{{package.name}}/pull-requests *] {% else %} - {%- endif %} + +|{% if package.upstreamfirst.test_yml %} yes [https://upstreamfirst.fedorainfracloud.org/{{package.name}}/tree/master *] {% else %} - {%- endif %} + +|{% if package.upstreamfirst.test_tags.classic %} yes {% else %} - {% endif %} + +|{% if package.upstreamfirst.test_tags.container %} yes {% else %} - {% endif %} + +|{% if package.upstreamfirst.test_tags.atomic %} yes {% else %} - {% endif %} + +{% endfor %} +{% raw %} +|} +{% endraw %} + + + + \ No newline at end of file diff --git a/statictic_page/stable_page/test-stats.py b/statictic_page/stable_page/test-stats.py new file mode 100755 index 0000000..018a4cb --- /dev/null +++ b/statictic_page/stable_page/test-stats.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3.6 +# -*- coding: utf-8 -*- + +# Copyright Red Hat Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# This file is part of Cockpit. + +import argparse +from tools import * + +from generate_file_upload_wiki import generate_file_upload_on_wiki + + +UPSTREAMFIRST_URL = config.UPSTREAMFIRST_URL +DIST_GIT_URL = config.DIST_GIT_URL + +package_json = {'name': '', + 'distgit': { + 'package_url': '', + 'test_yml': '', + 'missing': '', + 'pending': {'status': '', 'url': '', 'user': {}}, + 'test_tags': {'classic': '', 'container': '', 'atomic': ''}}, + 'upstreamfirst': {'test_yml': '', + 'package_url': '', + 'test_tags': {'classic': '', 'container': '', 'atomic': ''}}} + + +def wrapper_for_get_pull_requests(package): + """Function updates package_json with correct information""" + + raw_text = get_pull_requests(DIST_GIT_URL, package) + result = manage_pull_request(raw_text) + package_json['name'] = package + + if result is not None: + package_json['distgit']['pending']['url'] = result['url'] + package_json['distgit']['pending']['user'] = result['user'] + package_json['distgit']['pending']['status'] = True + else: + package_json['distgit']['pending']['url'] = '' + package_json['distgit']['pending']['user'] = '' + package_json['distgit']['pending']['status'] = False + + # Get upstreamfirst test-tags + upstream_test_tags = handle_test_tags(UPSTREAMFIRST_URL, package) + upstream_test_tags = test_tags_to_dict(upstream_test_tags) + package_json['upstreamfirst']['test_tags'] = upstream_test_tags + + upstream_url_to_test_yml = get_url_to_test_yml(UPSTREAMFIRST_URL, package) + if check_if_test_yml_exists(upstream_url_to_test_yml): + package_json['upstreamfirst']['test_yml'] = True + package_json['upstreamfirst']['package_url'] = upstream_url_to_test_yml + else: + package_json['upstreamfirst']['test_yml'] = False + package_json['upstreamfirst']['package_url'] = '' + + # Get distgit test-tags + dist_git_test_tags = handle_test_tags(DIST_GIT_URL, package) + dist_git_test_tags = test_tags_to_dict(dist_git_test_tags) + package_json['distgit']['test_tags'] = dist_git_test_tags + + dist_git_url_to_test_yml = get_url_to_test_yml(DIST_GIT_URL, package) + if check_if_test_yml_exists(dist_git_url_to_test_yml): + package_json['distgit']['test_yml'] = True + package_json['distgit']['package_url'] = dist_git_url_to_test_yml + else: + package_json['distgit']['test_yml'] = False + package_json['distgit']['package_url'] = '' + + # Write results to the file + write_results_to_the_file(package_json, config.FILE_PACKAGES) + + +def get_projects_list_from_url(site_url): + # Get all open pull requests + projects = get_all_projects(UPSTREAMFIRST_URL) + all_packages = get_projects_names(projects) + return all_packages + + +def create_file_with_packages_data(input_data): + # Create file with packages_data input data is a file or an URL. Filename is config.FILE_PACKAGES + if 'https://' in input_data: + all_packages = get_projects_list_from_url(input_data) + else: + all_packages = get_list_of_packages_from_the_file(input_data) + + for package in all_packages: + wrapper_for_get_pull_requests(package) + return + + +def main(): + parser = argparse.ArgumentParser(description='Gather stats about tests in dist-git') + parser.add_argument("--wikiout", metavar='FILE', default=None, help="Dump output to FILE in MediaWiki format.") + parser.add_argument("--reposlist", + metavar='REPOSFILE', + default=None, + help="File or URL with repos. Could be whether 'https://upstreamfirst.fedorainfracloud.org/' " + "or 'https://src.fedoraproject.org/'. One repo per line.", + required=True) + opts = parser.parse_args() + + # Prepare environment + delete_files_before_running([config.FILE_PACKAGES, opts.wikiout]) + + if opts.wikiout: + create_file_with_packages_data(opts.reposlist) + generate_file_upload_on_wiki(input_file=config.FILE_PACKAGES, output_file=opts.wikiout) + else: + create_file_with_packages_data(opts.reposlist) + + +if __name__ == '__main__': + main() diff --git a/statictic_page/stable_page/tools.py b/statictic_page/stable_page/tools.py new file mode 100644 index 0000000..f8551b6 --- /dev/null +++ b/statictic_page/stable_page/tools.py @@ -0,0 +1,267 @@ +import os +import re +import requests + +import config + + +UPSTREAMFIRST_URL = config.UPSTREAMFIRST_URL +DIST_GIT_URL = config.DIST_GIT_URL + + +def get_pull_requests(site_url, package): + """Get pull requests from site using API + + Args: + site_url (string): url of the site + for example: 'https://upstreamfirst.fedorainfracloud.org/' + package (string): name of the package + + Returns: + response (json): info about PR + """ + site = site_url + 'api/0/rpms/' + package + '/pull-requests' + + response = requests.get(site) + return response.json() + + +def manage_pull_request(response_json): + """Checks if PR corresponds to the test PR and open. + + Args: + response_json (json): info about PR + + Returns: + (json): {'user': , 'url': } + """ + try: + if response_json['total_requests'] > 0: + for request in response_json['requests']: + if ('Add CI tests' or 'test' in request['title']) and (request['status'] == 'Open'): + pull_req_url = DIST_GIT_URL + request['project']['url_path'] + '/pull-requests' + return {'user': request['user'], 'url': pull_req_url} + + except Exception: + # {'error': 'Project not found', 'error_code': 'ENOPROJECT'} + return + + +def get_all_projects(site_url): + """Get all projects from site using API + + Args: + site_url (string): url of the site + for example: 'https://upstreamfirst.fedorainfracloud.org/' + + Returns: + response (json): all projects + """ + site = site_url + 'api/0/projects' + + response = requests.get(site) + return response.json() + + +def get_projects_names(json_response): + """Get projects names from the all projects + + Args: + json_response (json): all projects + + Returns: + projects_names (list of strings) names of projects + """ + + projects_names = [] + + for project in json_response['projects']: + project_name = project['fullname'] + if 'forks/' not in project_name: + projects_names.append(project_name) + + return projects_names + + +def get_projects_url_patches(json_response): + """Get projects url patches""" + + projects_url_patches = [] + + for project in json_response['projects']: + project_name = project['url_path'] + + projects_url_patches.append(project_name) + + return projects_url_patches + + +def get_test_yaml_from_site(site_url, package, test_file='tests.yml'): + """Get test.yaml from the site + + Args: + site_url (string): url of the site + for example: 'https://upstreamfirst.fedorainfracloud.org/' + package (string): name of the package + test_file (string): file to get from site. + + Returns: + test.yaml (raw string) + """ + + if 'upstreamfirst' in site_url: + test_file_url = site_url + package + '/raw/master/f/' + test_file + elif 'fedoraproject' in site_url: + test_file_url = site_url + '/rpms/' + package + '/raw/master/f/tests/' + test_file + else: + return + + response = requests.get(test_file_url) + return response.text + + +def get_url_to_test_yml(site_url, package): + """Get url to the test.yml file + + Args: + site_url (string): url of the site + for example: 'https://upstreamfirst.fedorainfracloud.org/' + package (string): name of the package + + Returns: + url (string) + """ + if 'upstreamfirst' in site_url: + test_file_url = site_url + package + '/blob/master/f/tests.yml' + elif 'fedoraproject' in site_url: + test_file_url = site_url + 'rpms/' + package + '/blob/master/f/tests/tests.yml' + else: + return + + return test_file_url + + +def check_if_test_yml_exists(test_file_url): + """Checks if tesy.yml exists + + Args: + test_file_url (string): url of the site + + Returns: + True/False (bool) if response.status_code + """ + response = requests.get(test_file_url) + if response.status_code == 200: + return True + else: + return False + + +def get_test_tags(raw_text): + """Just returns existed test-tags + + Args: + raw_text (string): raw text output from the requests + + Returns: + tags (list if strings) + """ + test_tags = [] + + for tag in ['classic', 'container', 'atomic']: + if tag in raw_text: + test_tags.append(tag) + + return test_tags + + +def handle_test_tags(site_url, package): + """Function grts new patch to the test.yaml if existing test.yaml includes test file + + Args: + site_url (string): url of the site + for example: 'https://upstreamfirst.fedorainfracloud.org/' + package (string): name of the package + + Returns: + tags (list of strings) + """ + + raw_text = get_test_yaml_from_site(site_url, package) + + if 'Page not found' in raw_text: + return [] + + test_tags = get_test_tags(raw_text) + + if not test_tags: + new_test_file = re.findall(r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', raw_text) + + if new_test_file: + raw_text = get_test_yaml_from_site(site_url, package, new_test_file[-1]) + test_tags = get_test_tags(raw_text) + + return test_tags + + +def test_tags_to_dict(test_tags): + """Convert test-tags list to the dictionary""" + + tags_dict = {'classic': False, 'container': False, 'atomic': False} + try: + for tag in test_tags: + tags_dict[tag] = True + + except Exception: + pass + return tags_dict + + +def get_list_of_packages_from_the_file(file): + """Reads list of packages from a file + + Args: + file (file): file to get packages from + + Returns: + packages (list of packages) + """ + with open(file) as packages_file: + return packages_file.read().splitlines() + + +def write_results_to_the_file(result, filename='results.txt'): + """Write results to the file + + Args: + result (json): package's info + filename (string): name of the file to write packages + + Returns: + packages (list of packages) + """ + with open(filename, 'a') as my_file: + my_file.write('{}\n'.format(result)) + + my_file.close() + return filename + + +def delete_files_before_running(files_to_delete): + """Function deletes (cleanup) files with data + + Args: + files_to_delete (list): list of files to delete + + Returns: + (None) if everything is alright + """ + for file_name in files_to_delete: + try: + os.remove(file_name) + except OSError: + pass + + +if __name__ == '__main__': + exit(0)