From cb7b6a319354facfcdeb0e8e0038756e491f81f6 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Feb 15 2018 14:11:19 +0000 Subject: [PATCH 1/5] Set a default config file in /etc --- diff --git a/deploy/sysconfig b/deploy/sysconfig index fc020be..744dcdb 100644 --- a/deploy/sysconfig +++ b/deploy/sysconfig @@ -1,2 +1,2 @@ -HUBS_CONFIG=/etc/fedora-hubs/config.py +# HUBS_CONFIG=/etc/fedora-hubs/hubs.py LOGGING_CONFIG=/etc/fedora-hubs/logging.ini diff --git a/fedora-hubs.spec b/fedora-hubs.spec index 1cbe815..d647db5 100644 --- a/fedora-hubs.spec +++ b/fedora-hubs.spec @@ -155,8 +155,6 @@ EOF # Environment file mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/sysconfig cat > $RPM_BUILD_ROOT%{_sysconfdir}/sysconfig/%{name} << EOF -USER=%{username} -GROUP=%{username} HUBS_CONFIG=%{_sysconfdir}/%{name}/hubs.py LOGGING_CONFIG=%{_sysconfdir}/%{name}/logging.ini EOF @@ -210,7 +208,7 @@ done %files %license LICENSE -%doc README.rst +%doc README.rst deploy/nginx.conf %{python3_sitelib}/* %{_unitdir}/*.service %{_bindir}/fedora-hubs diff --git a/hubs/app.py b/hubs/app.py index ef00df2..55f28fb 100644 --- a/hubs/app.py +++ b/hubs/app.py @@ -31,6 +31,8 @@ logging.basicConfig() app.config.from_object('hubs.default_config') if 'HUBS_CONFIG' in os.environ: app.config.from_envvar('HUBS_CONFIG') +elif os.path.exists("/etc/fedora-hubs/hubs.py"): + app.config.from_pyfile("/etc/fedora-hubs/hubs.py") fedmsg_config = get_fedmsg_config() From c03a858b601d63146d9f96937deae89ff0e3a1d0 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Feb 15 2018 14:11:19 +0000 Subject: [PATCH 2/5] Fix the command line parameters --- diff --git a/hubs/fas/scripts.py b/hubs/fas/scripts.py index 7069c13..0cafa10 100644 --- a/hubs/fas/scripts.py +++ b/hubs/fas/scripts.py @@ -15,17 +15,17 @@ from .api import sync_team_hub, sync_team_hub_roles @click.command("create-team-from-fas") @click.argument("name") -def create_team_from_fas(hub_name): +def create_team_from_fas(name): """Create the team hub NAME from FAS.""" fedmsg_config = get_fedmsg_config() hubs.database.init(fedmsg_config['hubs.sqlalchemy.uri']) - hub = Hub.by_name(hub_name, "team") + hub = Hub.by_name(name, "team") if hub is not None: print("This hub already exists.") return fas_client = FASClient() try: - fas_client.group_by_name(hub_name) + fas_client.group_by_name(name) except AppError: print("Could not find this group in FAS.") return @@ -33,24 +33,24 @@ def create_team_from_fas(hub_name): with hubs.app.app.app_context(): db = hubs.database.Session() hubs.app.create_task_queue() - Hub.create_group_hub(hub_name, "") + Hub.create_group_hub(name, "") db.commit() - print("Hub {} created!".format(hub_name)) + print("Hub {} created!".format(name)) print("It will be synced from FAS in the background.") @click.command("sync-teams-from-fas") @click.option("--roles/--no-roles", default=False, help="Sync the roles.") @click.argument("name", nargs=-1) -def sync_teams_from_fas(roles, names): +def sync_teams_from_fas(roles, name): """Sync all the team hubs NAMEs from FAS.""" fedmsg_config = get_fedmsg_config() hubs.database.init(fedmsg_config['hubs.sqlalchemy.uri']) - if not names: + if not name: hubs_to_sync = Hub.query.filter_by(hub_type="team").all() else: hubs_to_sync = [] - for hub_name in names: + for hub_name in name: hub = hubs.models.Hub.by_name(hub_name, "team") if hub is None: print("The team hub {} does not exist.".format(hub_name)) From d792d354180d900e320871b0950dfdbc065fe7d6 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Feb 15 2018 14:11:19 +0000 Subject: [PATCH 3/5] Integrate the cache scripts --- diff --git a/check-cache-coverage.py b/check-cache-coverage.py deleted file mode 100755 index bbe9524..0000000 --- a/check-cache-coverage.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python -""" cache coverage checker. - -Runs through all possible cache keys and checks if there is a corresponding -value already cached for it. - -Useful as a sanity check during development. - -Could make a good nagios check one day. If the cache coverage is below 50%, -then the site will be very slow. We should run some sort of script that -forcibly re-warms the cache. - -Authors: Ralph Bean -""" - -from __future__ import unicode_literals, print_function - -import sys - -import hubs.app -import hubs.widgets -import hubs.models -from hubs.database import Session - -db = Session() -empty, full = 0, 0 - -# Register widgets -hubs.widgets.registry.register_list(hubs.app.app.config["WIDGETS"]) - -for w_instance in db.query(hubs.models.Widget).all(): - if not w_instance.enabled: - continue - for fn_name, fn_class in w_instance.module.get_cached_functions().items(): - if fn_class(w_instance).is_cached(): - full += 1 - else: - empty += 1 - sys.stdout.write('.') - sys.stdout.flush() -sys.stdout.write('\n') -sys.stdout.flush() - -total = empty + full -print(full, "cache values found. ", empty, "are missing.") -print(full / float(total) * 100, "percent cache coverage.") diff --git a/hubs/commands.py b/hubs/commands.py index 3ad478a..505457c 100644 --- a/hubs/commands.py +++ b/hubs/commands.py @@ -3,6 +3,7 @@ import click from hubs.backend.triage import main as triage from hubs.backend.worker import main as worker from hubs.fas.scripts import create_team_from_fas, sync_teams_from_fas +from hubs.utils.cache import cache_list, cache_clean, coverage @click.group() @@ -14,3 +15,14 @@ cli.add_command(triage) cli.add_command(worker) cli.add_command(create_team_from_fas) cli.add_command(sync_teams_from_fas) + + +@cli.group() +def cache(): + """Cache-related operations.""" + pass + + +cache.add_command(cache_list) +cache.add_command(cache_clean) +cache.add_command(coverage) diff --git a/hubs/utils/cache.py b/hubs/utils/cache.py index f087b64..1df583d 100644 --- a/hubs/utils/cache.py +++ b/hubs/utils/cache.py @@ -7,6 +7,7 @@ Attributes: from __future__ import unicode_literals +import click import dogpile import dogpile.cache @@ -28,3 +29,82 @@ def _get_cache(): cache = _get_cache() + + +def _setup_widgets_registry(): + import hubs.app + from hubs.widgets import registry + registry.register_list(hubs.app.app.config["WIDGETS"]) + + +@click.command("list") +def cache_list(): + """List widgets for which there is cached data.""" + from hubs.models import Widget + + _setup_widgets_registry() + + for w_instance in Widget.query.all(): + if not w_instance.enabled: + continue + widget = w_instance.module + for fn_name, fn_class in widget.get_cached_functions().items(): + if fn_class(w_instance).is_cached(): + print(('- Widget cached {inst.hub_id} (#{inst.idx}) ' + 'in {inst.plugin}').format(inst=w_instance)) + + +@click.command("clean") +@click.argument("widget", nargs=-1) +def cache_clean(widget): + """Clean the specified WIDGETs (id or name).""" + from hubs.models import Widget + + _setup_widgets_registry() + + count = 0 + for w in widget: + widget_instance = Widget.query.get(w) + + if widget_instance is None: + widget_instances = Widget.query.filter_by( + plugin=w).all() + else: + widget_instances = [widget_instance] + + if not widget_instances: + print('No widget found for {0}'.format(w)) + + for widget_instance in widget_instances: + print('- Removing cached {0} (#{1}) in {2}'.format( + widget_instance.hub_id, + widget_instance.idx, + widget_instance.plugin)) + functions = widget_instance.module.get_cached_functions().values() + for fn_class in functions: + fn_class(widget_instance).invalidate() + count += 1 + print("Cleaned {} widget caches.".format(count)) + + +@click.command() +def coverage(): + """Check the cache coverage.""" + from hubs.models import Widget + + _setup_widgets_registry() + empty, full = 0, 0 + + for w_instance in Widget.query.all(): + if not w_instance.enabled: + continue + functions = w_instance.module.get_cached_functions() + for fn_name, fn_class in functions.items(): + if fn_class(w_instance).is_cached(): + full += 1 + else: + empty += 1 + total = empty + full + print("{full} cached values found, {empty} are missing.".format( + full=full, empty=empty)) + print("{:.2f} percent cache coverage.".format(full / float(total) * 100)) diff --git a/smart_cache_invalidator.py b/smart_cache_invalidator.py deleted file mode 100755 index c1faa59..0000000 --- a/smart_cache_invalidator.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python - -""" -Small script to remove the cache of one or more widgets while keeping the -other ones un-touched. - -This is really useful for development purposes as it allow seeing changes -made to a widget without dropping the entire cache database. -""" - -from __future__ import print_function, unicode_literals - -import argparse - -import hubs.app -import hubs.models -import hubs.widgets.base -from hubs.database import Session - - -db = Session() -# Register widgets -hubs.widgets.registry.register_list(hubs.app.app.config["WIDGETS"]) - - -def do_list(args): - ''' List the different widget for which there is data cached. ''' - for w_instance in db.query(hubs.models.Widget).all(): - if not w_instance.enabled: - continue - widget = w_instance.module - for fn_name, fn_class in widget.get_cached_functions().items(): - if fn_class(w_instance).is_cached(): - print(('- Widget cached {inst.hub_id} (#{inst.idx}) ' - 'in {inst.plugin}').format(inst=w_instance)) - - -def do_clean(args): - ''' Clean the widget for which there is data cached. ''' - for widget in args.widgets: - widget_instance = hubs.models.Widget.get(widget) - - if widget_instance is None: - widget_instances = db.query(hubs.models.Widget).filter_by( - plugin=widget).all() - else: - widget_instances = [widget_instance] - - if not widget_instances: - print('No widget found for {0}'.format(widget)) - - for widget_instance in widget_instances: - print('- Removing cached {0} (#{1}) in {2}'.format( - widget_instance.hub_id, - widget_instance.idx, - widget_instance.plugin)) - functions = widget_instance.module.get_cached_functions().values() - for fn_class in functions: - fn_class(widget_instance).invalidate() - - -def setup_parser(): - ''' - Set the main arguments. - ''' - parser = argparse.ArgumentParser(prog="smart_cache_invalidator") - subparsers = parser.add_subparsers(title='actions') - - # List - parser_list = subparsers.add_parser( - 'list', - help='List the different widgets for which there is cached data') - parser_list.set_defaults(func=do_list) - - # Clean - parser_clean = subparsers.add_parser( - 'clean', help='Clean the specified widget') - parser_clean.add_argument( - 'widgets', nargs="+", - help="Identifier or name of the one or moe widgets to clean") - parser_clean.set_defaults(func=do_clean) - - return parser - - -def main(): - ''' Main function ''' - return_code = 0 - - # Set up parser for global args - parser = setup_parser() - - # Parse the commandline - try: - arg = parser.parse_args() - except argparse.ArgumentTypeError as err: - print("\nError: {0}".format(err)) - return 1 - - try: - arg.func(arg) - except KeyboardInterrupt: - print("\nInterrupted by user.") - return_code = 2 - except Exception as err: - print('Error: {0}'.format(err)) - return_code = 3 - - return return_code - - -if __name__ == '__main__': - main() From 5bfa88cf54ea603f6ba3ea5f8938d334811c780a Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Feb 15 2018 14:16:04 +0000 Subject: [PATCH 4/5] Put FAS scripts under their own subcommand --- diff --git a/hubs/commands.py b/hubs/commands.py index 505457c..a74edcd 100644 --- a/hubs/commands.py +++ b/hubs/commands.py @@ -2,7 +2,7 @@ import click from hubs.backend.triage import main as triage from hubs.backend.worker import main as worker -from hubs.fas.scripts import create_team_from_fas, sync_teams_from_fas +from hubs.fas.scripts import create_team, sync_teams from hubs.utils.cache import cache_list, cache_clean, coverage @@ -13,8 +13,6 @@ def cli(): cli.add_command(triage) cli.add_command(worker) -cli.add_command(create_team_from_fas) -cli.add_command(sync_teams_from_fas) @cli.group() @@ -26,3 +24,13 @@ def cache(): cache.add_command(cache_list) cache.add_command(cache_clean) cache.add_command(coverage) + + +@cli.group() +def fas(): + """FAS-related operations.""" + pass + + +fas.add_command(create_team) +fas.add_command(sync_teams) diff --git a/hubs/fas/scripts.py b/hubs/fas/scripts.py index 0cafa10..21f86ab 100644 --- a/hubs/fas/scripts.py +++ b/hubs/fas/scripts.py @@ -13,9 +13,9 @@ from .fasclient import FASClient from .api import sync_team_hub, sync_team_hub_roles -@click.command("create-team-from-fas") +@click.command("create-team") @click.argument("name") -def create_team_from_fas(name): +def create_team(name): """Create the team hub NAME from FAS.""" fedmsg_config = get_fedmsg_config() hubs.database.init(fedmsg_config['hubs.sqlalchemy.uri']) @@ -39,10 +39,10 @@ def create_team_from_fas(name): print("It will be synced from FAS in the background.") -@click.command("sync-teams-from-fas") +@click.command("sync-teams") @click.option("--roles/--no-roles", default=False, help="Sync the roles.") @click.argument("name", nargs=-1) -def sync_teams_from_fas(roles, name): +def sync_teams(roles, name): """Sync all the team hubs NAMEs from FAS.""" fedmsg_config = get_fedmsg_config() hubs.database.init(fedmsg_config['hubs.sqlalchemy.uri']) From 76acd27d36e9f58866d8302523801058f4335f57 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Feb 15 2018 14:21:06 +0000 Subject: [PATCH 5/5] Put deamon processes under their own subcommand --- diff --git a/ansible/roles/hubs/templates/honcho-procfile b/ansible/roles/hubs/templates/honcho-procfile index 7be0312..77692da 100644 --- a/ansible/roles/hubs/templates/honcho-procfile +++ b/ansible/roles/hubs/templates/honcho-procfile @@ -1,6 +1,6 @@ web: /usr/bin/flask-3 run --host 0.0.0.0 --port 5000 -triage: fedora-hubs triage -worker: fedora-hubs worker +triage: fedora-hubs run triage +worker: fedora-hubs run worker sse: /usr/bin/twistd-3 -l - --pidfile= -n hubs-sse fedmsg_hub: /usr/bin/fedmsg-hub-3 fedmsg_relay: /usr/bin/fedmsg-relay-3 diff --git a/deploy/fedora-hubs-triage@.service b/deploy/fedora-hubs-triage@.service index 1f5fb02..724d955 100644 --- a/deploy/fedora-hubs-triage@.service +++ b/deploy/fedora-hubs-triage@.service @@ -4,7 +4,7 @@ After=network.target Documentation=https://pagure.io/fedora-hubs/ [Service] -ExecStart=/usr/bin/fedora-hubs triage +ExecStart=/usr/bin/fedora-hubs run triage EnvironmentFile=/etc/sysconfig/fedora-hubs Type=simple User=hubs diff --git a/deploy/fedora-hubs-worker@.service b/deploy/fedora-hubs-worker@.service index 867180a..8b1854b 100644 --- a/deploy/fedora-hubs-worker@.service +++ b/deploy/fedora-hubs-worker@.service @@ -4,7 +4,7 @@ After=network.target Documentation=https://pagure.io/fedora-hubs/ [Service] -ExecStart=/usr/bin/fedora-hubs worker +ExecStart=/usr/bin/fedora-hubs run worker EnvironmentFile=/etc/sysconfig/fedora-hubs Type=simple User=hubs diff --git a/hubs/commands.py b/hubs/commands.py index a74edcd..3ccd7db 100644 --- a/hubs/commands.py +++ b/hubs/commands.py @@ -11,8 +11,14 @@ def cli(): pass -cli.add_command(triage) -cli.add_command(worker) +@cli.group() +def run(): + """Run daemon processes.""" + pass + + +run.add_command(triage) +run.add_command(worker) @cli.group() diff --git a/systemd/hubs-triage@.service b/systemd/hubs-triage@.service index db5b9c8..b918cd4 100644 --- a/systemd/hubs-triage@.service +++ b/systemd/hubs-triage@.service @@ -4,7 +4,7 @@ After=network.target Documentation=https://pagure.io/fedora-hubs/ [Service] -ExecStart=/usr/bin/fedora-hubs triage +ExecStart=/usr/bin/fedora-hubs run triage WorkingDirectory=/srv/hubs/fedora-hubs/ Environment=HUBS_CONFIG=/srv/hubs/fedora-hubs/config Type=simple diff --git a/systemd/hubs-worker@.service b/systemd/hubs-worker@.service index d253779..c507023 100644 --- a/systemd/hubs-worker@.service +++ b/systemd/hubs-worker@.service @@ -4,7 +4,7 @@ After=network.target Documentation=https://pagure.io/fedora-hubs/ [Service] -ExecStart=/usr/bin/fedora-hubs worker +ExecStart=/usr/bin/fedora-hubs run worker Environment=HUBS_CONFIG=/srv/git/fedora-hubs/config Type=simple User=root