From 670020c95a29a0a2acf74ee683a272e0427e2612 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Oct 01 2019 19:46:29 +0000 Subject: Use flask and gunicorn instead of aiohttp. This commit moves mdapi from the aiohttp framework to flask. This is done since there is little interest in using aiohttp because all the call to sqlite are io blocking. There is no asyncio driver for sqlalchemy and sqlite. Also performance with Flask and 4 gunicorn workers are better than the performances achieved with aiohttp. Signed-off-by: Clement Verna --- diff --git a/Dockerfile b/Dockerfile index 8fce400..b6a8ffb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,9 +6,10 @@ LABEL maintainer "Clément Verna " EXPOSE 8080 -RUN dnf -y install python3-aiohttp python3-werkzeug python3-requests python3-sqlalchemy python3-fedora-messaging +RUN dnf -y install python3-flask python3-requests python3-sqlalchemy python3-fedora-messaging python3-gunicorn USER 1001 ENV MDAPI_CONFIG=/etc/mdapi/mdapi.cfg +ENV PYTHONPATH=/code COPY . /code -ENTRYPOINT ["/code/mdapi-run"] +ENTRYPOINT ["gunicorn-3", "-w", "4", "-b", "0.0.0.0:8080", "--log-level=debug", "mdapi:app"] diff --git a/README.rst b/README.rst index 98fc0e0..11a7d1b 100644 --- a/README.rst +++ b/README.rst @@ -14,23 +14,12 @@ Development If you wish to set up a development instance of this project, follow these steps: -* Install virtualenvwrapper +* Create and activate the virtualenv :: - dnf install python-virtualenvwrapper python3-devel openssl-devel - -* Get the mkvirtualenv shell function - -:: - - . /etc/profile.d/virtualenvwrapper.sh - -* Create the virtualenv - -:: - - mkvirtualenv mdapi -p python3 + python3 -m venv .venv + source .venv/bin/activate * Install the dependencies @@ -42,19 +31,17 @@ development instance of this project, follow these steps: :: - ./mdapi-get_repo_md mdapi/default_config.py + ./mdapi-get_repo_md mdapi/mdapi.cfg * Start the server :: - ./mdapi-run - -* If you need to reactivate the virtual env later, use workon: + gunicorn-3 mdapi:app +* Run the test suite :: - workon mdapi - + tox **Note:** This project is python3 only diff --git a/mdapi-get_repo_md b/mdapi-get_repo_md index 0fb31f2..d12f013 100755 --- a/mdapi-get_repo_md +++ b/mdapi-get_repo_md @@ -49,12 +49,13 @@ import sys import requests -from sqlalchemy import text +import sqlalchemy as sa +from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm import scoped_session + from fedora_messaging.api import Message, publish from fedora_messaging.exceptions import PublishReturned, ConnectionException -import mdapi.lib as mdapilib - KOJI_REPO = 'https://kojipkgs.fedoraproject.org/repos/' PKGDB2_URL = 'https://admin.fedoraproject.org/pkgdb/' DL_SERVER = 'http://dl.fedoraproject.org' @@ -149,6 +150,22 @@ queries = { } +@contextlib.contextmanager +def session_manager(db_url, debug=False, pool_recycle=3600): + """ A handy context manager for our sessions. """ + engine = sa.create_engine( + db_url, echo=debug, pool_recycle=pool_recycle) + session = scoped_session(sessionmaker(bind=engine)) + try: + yield session + session.commit() + except: + session.rollback() + raise + finally: + session.close() + + def list_branches(status='Active'): ''' Return the list of Fedora branches corresponding to the given status. @@ -199,7 +216,7 @@ def compare_dbs(name, db1, db2, cache1, cache2): print('%s Comparing %s and %s' % (name.ljust(padding), db1, db2)) def get_table_names(uri): - with mdapilib.session_manager('sqlite:///' + uri) as session: + with session_manager('sqlite:///' + uri) as session: for name in session.connection().engine.table_names(): if name == 'db_info': continue @@ -213,8 +230,8 @@ def compare_dbs(name, db1, db2, cache1, cache2): return name.split('(')[0] def get_all_rows(uri, table, cache): - query = text(queries.get(table, default_query).format(table=table)) - with mdapilib.session_manager('sqlite:///' + uri) as session: + query = sa.text(queries.get(table, default_query).format(table=table)) + with session_manager('sqlite:///' + uri) as session: engine = session.connection().engine for i, row in enumerate(engine.execute(query)): if table in cache_dependant_tables: @@ -232,8 +249,8 @@ def compare_dbs(name, db1, db2, cache1, cache2): def build_cache(uri, cache): - query = text(packages_cache_builder.format(table=table)) - with mdapilib.session_manager('sqlite:///' + uri) as session: + query = sa.text(packages_cache_builder.format(table=table)) + with session_manager('sqlite:///' + uri) as session: engine = session.connection().engine for pkgId, pkgname in engine.execute(query): cache[pkgId] = pkgname diff --git a/mdapi-run b/mdapi-run deleted file mode 100755 index d6907f3..0000000 --- a/mdapi-run +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 - -from mdapi.server import main - -main() diff --git a/mdapi.service b/mdapi.service deleted file mode 100644 index c480956..0000000 --- a/mdapi.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=mdapi, a simple API to expose metadata from RPM repositories -Documentation=https://pagure.io/mdapi - -[Service] -Environment=MDAPI_CONFIG=/etc/mdapi/mdapi.cfg -ExecStart=/usr/bin/mdapi-run -Type=simple -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/mdapi.spec b/mdapi.spec deleted file mode 100644 index 47d2a3d..0000000 --- a/mdapi.spec +++ /dev/null @@ -1,202 +0,0 @@ -Name: mdapi -Version: 2.10.5 -Release: 1%{?dist} -Summary: A simple API to query the metadata of the repositories - -License: GPLv2+ -URL: https://pagure.io/mdapi -Source0: https://pagure.io/releases/mdapi/%{name}-%{version}.tar.gz - -BuildArch: noarch - -BuildRequires: python3-aiohttp -BuildRequires: python3-requests -BuildRequires: python3-setuptools -BuildRequires: python3-sqlalchemy -BuildRequires: python3-werkzeug -BuildRequires: python3-devel -BuildRequires: systemd - -Requires: python3-aiohttp -Requires: python3-multidict -Requires: python3-requests -Requires: python3-setuptools -Requires: python3-sqlalchemy -Requires: python3-werkzeug - -Requires(post): systemd -Requires(preun): systemd -Requires(postun): systemd - - -%description -Small web and asynchronous application serving the metadata of the Fedora -repositories - - -%prep -%setup -q - -%build -%{__python3} setup.py build - - -%install -%{__python3} setup.py install -O1 --skip-build --root=%{buildroot} - -# Install the systemd service file -mkdir -p $RPM_BUILD_ROOT/%{_unitdir} -install -m 644 mdapi.service $RPM_BUILD_ROOT/%{_unitdir}/mdapi.service - - -%post -%systemd_post mdapi.service - -%preun -%systemd_preun mdapi.service - -%postun -%systemd_postun_with_restart mdapi.service - - -%files -%doc COPYING -%{python3_sitelib}/mdapi/ -%{python3_sitelib}/mdapi*.egg-info -%{_bindir}/mdapi-get_repo_md -%{_bindir}/mdapi-run -%{_unitdir}/mdapi.service - - -%changelog -* Fri Feb 09 2018 Pierre-Yves Chibon - 2.10.5-1 -- Update to 2.10.5 -- Fix the srcpkg endpoint by making it search for a direct match before trying - the sub-packages - -* Fri Feb 09 2018 Pierre-Yves Chibon - 2.10.4-1 -- Update to 2.10.4 -- Fix the content_type header on JSONP output - -* Wed Jan 10 2018 Pierre-Yves Chibon - 2.10.3-1 -- Update to 2.10.3 -- Always initialize the output variable -- Fix getting package by their source name - -* Tue Aug 08 2017 Pierre-Yves Chibon - 2.10.2-1 -- Update to 2.10.2 -- Fix importing MultiDict from multidict instead of aiohttp - -* Tue Aug 08 2017 Pierre-Yves Chibon - 2.10.1-1 -- Update to 2.10.1 -- Fix the srcpkg endpoints - -* Wed Aug 02 2017 Pierre-Yves Chibon - 2.10-1 -- Update to 2.10 -- Fix displaying the proper basename -- Expose the upstream URL as present in the metadata in the JSON returned -- Fix the JSONP support of the branches endpoint - -* Wed Jul 12 2017 Pierre-Yves Chibon - 2.9-1 -- Update to 2.9 -- Add JSONP support to the application for easier integration with other apps - -* Tue May 02 2017 Pierre-Yves Chibon - 2.8-1 -- Update to 2.8 -- Fix and improve the README -- Set the content_type and charset of the index page -- Download the source repository metadata as src_ and expose them -- Order the list of branches returned - -* Mon Sep 12 2016 Pierre-Yves Chibon - 2.7-1 -- Update to 2.7 -- Add the possibility to query package by the name of their source package -- Return flags (Igor Gnatenko) -- Fix querying packages by their source package name (Igor Gnatenko) - -* Thu May 12 2016 Pierre-Yves Chibon - 2.6-1 -- Update to 2.6 -- Fix get_repo_md to handle gzip correctly (Patrick Uiterwijk) -- Let the mdapi web app to return JSON with the correct mimetype (Patrick - Uiterwijk) -- Adjust get_repo_md for the new URL structure on the mirrors - -* Wed Mar 02 2016 Pierre-Yves Chibon - 2.5-1 -- Update to 2.5 -- Chain the method as coroutines making the process more asynchronous and a - little bit faster -- Streamline the fedmsg diff publication to reduce the amount of data sent per - message (especially for new repo where the diff concerns all the packages) - -* Mon Feb 29 2016 Pierre-Yves Chibon - 2.4-1 -- Update to 2.4 -- Adjust copyright year -- Expand mdapi to return dependencies information (conflicts, enhances, - recommends, suggests, supplements, requires, provides and obsoletes) -- Improve the cron script to retry after 30 seconds waiting, 3 times max if it - fails to get info for a repo - -* Wed Jan 13 2016 Pierre-Yves Chibon - 2.3-1 -- Update to 2.3 -- Return pretty JSON when the accept header is text/html (Till Maas) -- Order packages by their Epoch-Version-Release and return only the most recent -- Mention deployment URL in the README -- Adjust the JSON key from `files` to `changelogs` to reflect the content -- Rely on urllib and aiohttp.MultiDict to do the url arguments parsing -- Fix using ?pretty=False in a browser -- Drop the dependency on simplejson - -* Tue Nov 24 2015 Pierre-Yves Chibon - 2.2.3-1 -- Update to 2.2.3 -- Fix the branches endpoint (un-instantiated variable) - -* Mon Nov 23 2015 Pierre-Yves Chibon - 2.2.2-1 -- Update to 2.2.2 -- Fix accessing the configuration to adjust the link on the front page - -* Sun Nov 22 2015 Pierre-Yves Chibon - 2.2.1-1 -- Update to 2.2.1 -- Fix the links in the front page with it's accessed without trailing slash - (Patrick Uiterwijk) - -* Thu Nov 19 2015 Pierre-Yves Chibon - 2.2-1 -- Update to 2.2 -- Fix typo in the cron job - -* Thu Nov 19 2015 Pierre-Yves Chibon - 2.1-1 -- Update to 2.1 -- Drop un-used import -- Fix prettifying the JSON only on demand - -* Thu Nov 19 2015 Pierre-Yves Chibon - 2.0-1 -- Update to 2.0 -- Fix typos on the front page -- Add the dependencies (Requires, Obsoletes, Provides, Conflicts, Enhances, - Recommends, Suggests, Supplements) to the JSON returned -- Add the option to return a prettier JSON -- Add the possibility to disable checking the SSL cert - -* Tue Nov 10 2015 Pierre-Yves Chibon - 1.2.2-1 -- Update to 1.2.2 -- Be consistent about providing links on the front page - -* Tue Nov 10 2015 Pierre-Yves Chibon - 1.2.1-1 -- Update to 1.2.1 -- Fix typo in the routing - -* Tue Nov 10 2015 Pierre-Yves Chibon - 1.2-1 -- Update 1.2 -- Add the possibility to prefix the URLs at which mdapi answers -- Simplify the routing by listing them -- Fix the example and link in the documentation on the front page - -* Tue Nov 10 2015 Pierre-Yves Chibon - 1.1-1 -- Update to 1.1 -- Make the URLs to pkgdb, the koji repo and the download server configurable for - the cron job - -* Mon Nov 09 2015 Pierre-Yves Chibon - 1.0-1 -- Update to 1.0 - -* Tue Oct 27 2015 Pierre-Yves Chibon - 0.1-1 -- First package for Fedora diff --git a/mdapi/__init__.py b/mdapi/__init__.py index d0525f5..beb7514 100644 --- a/mdapi/__init__.py +++ b/mdapi/__init__.py @@ -22,86 +22,36 @@ ''' Top level of the mdapi aiohttp application. ''' -import functools -import json -import logging import os +import logging -import asyncio -import werkzeug -from aiohttp import web +from flask import Flask, jsonify, send_from_directory, abort import mdapi.lib as mdapilib -CONFIG = dict() -obj = werkzeug.import_string('mdapi.default_config') -for key in dir(obj): - if key.isupper(): - CONFIG[key] = getattr(obj, key) - - -if 'MDAPI_CONFIG' in os.environ and os.path.exists(os.environ['MDAPI_CONFIG']): - with open(os.environ['MDAPI_CONFIG']) as config_file: - exec(compile( - config_file.read(), os.environ['MDAPI_CONFIG'], 'exec'), CONFIG) - -indexfile = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'index.html') -INDEX = '' -with open(indexfile) as stream: - INDEX = stream.read() - INDEX = INDEX.replace('$PREFIX', CONFIG.get('PREFIX', '')) - - -_log = logging.getLogger(__name__) - - -def allows_jsonp(function): - ''' Add support for JSONP queries to the endpoint decorated. ''' +app = Flask(__name__) +app.config["DB_FOLDER"] = "/var/tmp" - @functools.wraps(function) - def wrapper(request, *args, **kwargs): - ''' Actually does the job with the arguments provided. +gunicorn_logger = logging.getLogger("gunicorn.error") +app.logger.handlers = gunicorn_logger.handlers +app.logger.setLevel(gunicorn_logger.level) - :arg request: the request that was called that we want to add JSONP - support to - :type request: aiohttp.web_request.Request - ''' - response = yield from function(request, *args, **kwargs) - url_arg = request.query - callback = url_arg.get('callback') - if callback and request.method == 'GET': - if isinstance(callback, list): - callback = callback[0] - response.mimetype = 'application/javascript' - response.content_type = 'application/javascript' - response.text = '%s(%s);' % (callback, response.text) - - return response - - return wrapper - - -@asyncio.coroutine def _get_pkg(branch, name=None, action=None, srcname=None): ''' Return the pkg information for the given package in the specified branch or raise an aiohttp exception. ''' - if (not name and not srcname) or (name and srcname): - raise web.HTTPBadRequest() - pkg = None wrongdb = False for repotype in ['updates-testing', 'updates', 'testing', None]: if repotype: dbfile = '%s/mdapi-%s-%s-primary.sqlite' % ( - CONFIG['DB_FOLDER'], branch, repotype) + app.config['DB_FOLDER'], branch, repotype) else: dbfile = '%s/mdapi-%s-primary.sqlite' % ( - CONFIG['DB_FOLDER'], branch) + app.config['DB_FOLDER'], branch) if not os.path.exists(dbfile): wrongdb = True @@ -109,41 +59,29 @@ def _get_pkg(branch, name=None, action=None, srcname=None): wrongdb = False - session = yield from mdapilib.create_session( + session = mdapilib.create_session( 'sqlite:///%s' % dbfile) if name: if action: - pkg = yield from mdapilib.get_package_by( + pkg = mdapilib.get_package_by( session, action, name) else: - pkg = yield from mdapilib.get_package(session, name) + pkg = mdapilib.get_package(session, name) elif srcname: - pkg = yield from mdapilib.get_package_by_src(session, srcname) + pkg = mdapilib.get_package_by_src(session, srcname) session.close() if pkg: break if wrongdb: - raise web.HTTPBadRequest() + abort(400) if not pkg: - raise web.HTTPNotFound() + abort(404) return (pkg, repotype) -def _get_pretty(request): - pretty = False - params = request.query - if params.get('pretty') in ['1', 'true']: - pretty = True - # Assume pretty if html is requested and pretty is not disabled - elif 'text/html' in request.headers.get('ACCEPT', ''): - pretty = True - return pretty - - -@asyncio.coroutine def _expand_pkg_info(pkgs, branch, repotype=None): ''' Return a JSON blob containing all the information we want to return for the provided package or packages. @@ -156,15 +94,15 @@ def _expand_pkg_info(pkgs, branch, repotype=None): for pkg in pkgs: out = pkg.to_json() dbfile = '%s/mdapi-%s%s-primary.sqlite' % ( - CONFIG['DB_FOLDER'], branch, '-%s' % repotype if repotype else '') + app.config['DB_FOLDER'], branch, '-%s' % repotype if repotype else '') - session = yield from mdapilib.create_session( + session = mdapilib.create_session( 'sqlite:///%s' % dbfile) # Fill in some extra info # Basic infos, always present regardless of the version of the repo for datatype in ['conflicts', 'obsoletes', 'provides', 'requires']: - data = yield from mdapilib.get_package_info( + data = mdapilib.get_package_info( session, pkg.pkgKey, datatype.capitalize()) if data: out[datatype] = [item.to_json() for item in data] @@ -174,7 +112,7 @@ def _expand_pkg_info(pkgs, branch, repotype=None): # New meta-data present for soft dependency management in RPM for datatype in [ 'enhances', 'recommends', 'suggests', 'supplements']: - data = yield from mdapilib.get_package_info( + data = mdapilib.get_package_info( session, pkg.pkgKey, datatype.capitalize()) if data: out[datatype] = [item.to_json() for item in data] @@ -183,7 +121,7 @@ def _expand_pkg_info(pkgs, branch, repotype=None): # Add the list of packages built from the same src.rpm if pkg.rpm_sourcerpm: - copkgs = yield from mdapilib.get_co_packages( + copkgs = mdapilib.get_co_packages( session, pkg.rpm_sourcerpm) out['co-packages'] = list(set([ cpkg.name for cpkg in copkgs @@ -199,248 +137,129 @@ def _expand_pkg_info(pkgs, branch, repotype=None): return output -@asyncio.coroutine -@allows_jsonp -def get_pkg(request): - _log.info('get_pkg %s', request) - branch = request.match_info.get('branch') - pretty = _get_pretty(request) - name = request.match_info.get('name') - pkg, repotype = yield from _get_pkg(branch, name) - - output = yield from _expand_pkg_info(pkg, branch, repotype) +@app.route("//pkg/") +def get_pkg(branch, name): + pkg, repotype = _get_pkg(branch, name) + output = _expand_pkg_info(pkg, branch, repotype) + return jsonify(output) - args = {} - if pretty: - args = dict(sort_keys=True, indent=4, separators=(',', ': ')) - output = web.Response( - body=json.dumps(output, **args).encode('utf-8'), - content_type='application/json') - return output +@app.route("//srcpkg/") +def get_src_pkg(branch, name): + pkg, repotype = _get_pkg(branch, srcname=name) + output = _expand_pkg_info(pkg, branch, repotype) + return jsonify(output) -@asyncio.coroutine -@allows_jsonp -def get_src_pkg(request): - _log.info('get_src_pkg %s', request) - branch = request.match_info.get('branch') - pretty = _get_pretty(request) - name = request.match_info.get('name') - pkg, repotype = yield from _get_pkg(branch, srcname=name) - output = yield from _expand_pkg_info(pkg, branch, repotype) - - args = {} - if pretty: - args = dict(sort_keys=True, indent=4, separators=(',', ': ')) - - return web.Response( - body=json.dumps(output, **args).encode('utf-8'), - content_type='application/json') - - -@asyncio.coroutine -@allows_jsonp -def get_pkg_files(request): - _log.info('get_pkg_files %s', request) - branch = request.match_info.get('branch') - name = request.match_info.get('name') - pretty = _get_pretty(request) - pkg, repotype = yield from _get_pkg(branch, name) +@app.route("//files/") +def get_pkg_files(branch, name): + pkg, repotype = _get_pkg(branch, name) dbfile = '%s/mdapi-%s%s-filelists.sqlite' % ( - CONFIG['DB_FOLDER'], branch, '-%s' % repotype if repotype else '') + app.config['DB_FOLDER'], branch, '-%s' % repotype if repotype else '') if not os.path.exists(dbfile): - raise web.HTTPBadRequest() + abort(400) - session2 = yield from mdapilib.create_session( + session2 = mdapilib.create_session( 'sqlite:///%s' % dbfile) - filelist = yield from mdapilib.get_files(session2, pkg.pkgId) + filelist = mdapilib.get_files(session2, pkg.pkgId) session2.close() output = { 'files': [fileinfo.to_json() for fileinfo in filelist], 'repo': repotype if repotype else 'release', } - args = {} - if pretty: - args = dict(sort_keys=True, indent=4, separators=(',', ': ')) - - return web.Response( - body=json.dumps(output, **args).encode('utf-8'), - content_type='application/json') + return jsonify(output) -@asyncio.coroutine -@allows_jsonp -def get_pkg_changelog(request): - _log.info('get_pkg_changelog %s', request) - branch = request.match_info.get('branch') - name = request.match_info.get('name') - pretty = _get_pretty(request) - pkg, repotype = yield from _get_pkg(branch, name) +@app.route("//changelog/") +def get_pkg_changelog(branch, name): + pkg, repotype = _get_pkg(branch, name) dbfile = '%s/mdapi-%s%s-other.sqlite' % ( - CONFIG['DB_FOLDER'], branch, '-%s' % repotype if repotype else '') + app.config['DB_FOLDER'], branch, '-%s' % repotype if repotype else '') if not os.path.exists(dbfile): - raise web.HTTPBadRequest() + abort(400) - session2 = yield from mdapilib.create_session( + session2 = mdapilib.create_session( 'sqlite:///%s' % dbfile) - changelogs = yield from mdapilib.get_changelog(session2, pkg.pkgId) + changelogs = mdapilib.get_changelog(session2, pkg.pkgId) session2.close() output = { 'changelogs': [changelog.to_json() for changelog in changelogs], 'repo': repotype if repotype else 'release', } - args = {} - if pretty: - args = dict(sort_keys=True, indent=4, separators=(',', ': ')) - return web.Response( - body=json.dumps(output, **args).encode('utf-8'), - content_type='application/json') + return jsonify(output) -@asyncio.coroutine -def list_branches(request): +@app.route("/branches") +def list_branches(): ''' Return the list of all branches currently supported by mdapi ''' - _log.info('list_branches: %s', request) - pretty = _get_pretty(request) output = sorted(list(set([ # Remove the front part `mdapi-` and the end part -.sqlite filename.replace('mdapi-', '').rsplit('-', 2)[0].replace( '-updates', '') - for filename in os.listdir(CONFIG['DB_FOLDER']) + for filename in os.listdir(app.config['DB_FOLDER']) if filename.startswith('mdapi') and filename.endswith('.sqlite') ]))) - args = {} - if pretty: - args = dict(sort_keys=True, indent=4, separators=(',', ': ')) + return jsonify(output) - response = web.Response(body=json.dumps(output, **args).encode('utf-8'), - content_type='application/json') - # The decorator doesn't work for this endpoint, so do it manually here - # I am not really sure what doesn't work but it seems this endpoint is - # returning an object instead of the expected generator despite it being - # flagged as an asyncio coroutine - url_arg = request.query - callback = url_arg.get('callback') - if callback and request.method == 'GET': - if isinstance(callback, list): - callback = callback[0] - response.mimetype = 'application/javascript' - response.content_type = 'application/javascript' - response.text = '%s(%s);' % (callback, response.text) - - return response - - -@asyncio.coroutine -@allows_jsonp -def process_dep(request, action): +def process_dep(branch, name, action): ''' Return the information about the packages having the specified action (provides, requires, obsoletes...) ''' - _log.info('process_dep %s: %s', action, request) - branch = request.match_info.get('branch') - pretty = _get_pretty(request) - name = request.match_info.get('name') - - try: - pkg, repotype = yield from _get_pkg(branch, name, action=action) - except: - raise web.HTTPBadRequest() - - output = yield from _expand_pkg_info(pkg, branch, repotype) - - args = {} - if pretty: - args = dict(sort_keys=True, indent=4, separators=(',', ': ')) - - return web.Response(body=json.dumps(output, **args).encode('utf-8'), - content_type='application/json') - - -@asyncio.coroutine -def get_provides(request): - return process_dep(request, 'provides') - - -@asyncio.coroutine -def get_requires(request): - return process_dep(request, 'requires') - + pkg, repotype = _get_pkg(branch, name, action=action) + output = _expand_pkg_info(pkg, branch, repotype) -@asyncio.coroutine -def get_obsoletes(request): - return process_dep(request, 'obsoletes') + return jsonify(output) -@asyncio.coroutine -def get_conflicts(request): - return process_dep(request, 'conflicts') +@app.route("//provides/") +def get_provides(branch, name): + return process_dep(branch, name, 'provides') -@asyncio.coroutine -def get_enhances(request): - return process_dep(request, 'enhances') +@app.route("//requires/") +def get_requires(branch, name): + return process_dep(branch, name, 'requires') -@asyncio.coroutine -def get_recommends(request): - return process_dep(request, 'recommends') +@app.route("//obsoletes/") +def get_obsoletes(branch, name): + return process_dep(branch, name, 'obsoletes') -@asyncio.coroutine -def get_suggests(request): - return process_dep(request, 'suggests') +@app.route("//conflicts/") +def get_conflicts(branch, name): + return process_dep(branch, name, 'conflicts') -@asyncio.coroutine -def get_supplements(request): - return process_dep(request, 'supplements') +@app.route("//enhances/") +def get_enhances(branch, name): + return process_dep(branch, name, 'enhances') -@asyncio.coroutine -def index(request): - _log.info('index %s', request) - return web.Response( - body=INDEX.encode('utf-8'), - content_type='text/html', - charset='utf-8') +@app.route("//recommends/") +def get_recommends(branch, name): + return process_dep(branch, name, 'recommends') -def _set_routes(app): - routes = [] - prefix = CONFIG.get('PREFIX', '') - if prefix: - routes.append(('', index)) +@app.route("//suggests/") +def get_suggests(branch, name): + return process_dep(branch, name, 'suggests') - routes.extend([ - ('/', index), - ('/branches', list_branches), - ('/{branch}/pkg/{name}', get_pkg), - ('/{branch}/srcpkg/{name}', get_src_pkg), - ('/{branch}/provides/{name}', get_provides), - ('/{branch}/requires/{name}', get_requires), - ('/{branch}/obsoletes/{name}', get_obsoletes), - ('/{branch}/conflicts/{name}', get_conflicts), +@app.route("//supplements/") +def get_supplements(branch, name): + return process_dep(branch, name, 'supplements') - ('/{branch}/enhances/{name}', get_enhances), - ('/{branch}/recommends/{name}', get_recommends), - ('/{branch}/suggests/{name}', get_suggests), - ('/{branch}/supplements/{name}', get_supplements), - ('/{branch}/files/{name}', get_pkg_files), - ('/{branch}/changelog/{name}', get_pkg_changelog), - ]) - for route in routes: - app.router.add_route('GET', prefix + route[0], route[1]) - return app +@app.route("/") +def index(): + return send_from_directory(directory="", filename="index.html") diff --git a/mdapi/index.html b/mdapi/index.html index 549af20..99f8401 100644 --- a/mdapi/index.html +++ b/mdapi/index.html @@ -68,7 +68,7 @@ at: /branches - /branches + /branches Note: @@ -88,7 +88,7 @@ by querying: So for example, for the kernel in rawhide: - /rawhide/pkg/kernel + /rawhide/pkg/kernel You can also retrieve information about a specific package on a specific @@ -98,7 +98,7 @@ branch via the name of its source package by querying: So for example, for the python-natsort in rawhide that only exists as src.rpm: - /rawhide/srcpkg/python-natsort + /rawhide/srcpkg/python-natsort Retrieve the list of files in a package @@ -111,7 +111,7 @@ specific branch by querying: So for example, for the kernel-core in rawhide: - /rawhide/files/kernel-core + /rawhide/files/kernel-core Retrieve the changelog of a package @@ -124,7 +124,7 @@ by querying: So for example, for the kernel in rawhide: - /rawhide/changelog/kernel + /rawhide/changelog/kernel Retrieve the packages having a specific property @@ -143,29 +143,29 @@ For example to retrieve the list of packages that require a specific package: Few examples: packages requiring R in rawhide: - /rawhide/requires/R - To see what R itself requires, check its information using: /rawhide/pkg/R + /rawhide/requires/R + To see what R itself requires, check its information using: /rawhide/pkg/R packages providing perl(SetupLog) in rawhide: - /rawhide/provides/perl(SetupLog) + /rawhide/provides/perl(SetupLog) packages obsoleting cabal2spec in rawhide: - rawhide/obsoletes/cabal2spec + rawhide/obsoletes/cabal2spec packages conflicting with mariadb in rawhide: - rawhide/conflicts/mariadb + rawhide/conflicts/mariadb packages enhancing httpd in rawhide: - rawhide/enhances/httpd + rawhide/enhances/httpd packages recommending flac in rawhide: - rawhide/recommends/flac + rawhide/recommends/flac packages suggesting R-tools in rawhide: - rawhide/suggests/R-tools + rawhide/suggests/R-tools packages supplementing `(hunspell and langpacks-fr)` in rawhide: - rawhide/supplements/(hunspell and langpacks-fr) + rawhide/supplements/(hunspell and langpacks-fr) diff --git a/mdapi/lib.py b/mdapi/lib.py index 2af05ab..725a273 100644 --- a/mdapi/lib.py +++ b/mdapi/lib.py @@ -23,10 +23,8 @@ MDAPI internal API to interact with the database. ''' -import contextlib import time -import asyncio import sqlalchemy as sa from sqlalchemy.orm import sessionmaker @@ -41,7 +39,6 @@ import mdapi.primary as primary RETRY_ATTEMPT = 3 -@asyncio.coroutine def create_session(db_url, debug=False, pool_recycle=3600): """ Create the Session object to use to query the database. @@ -60,22 +57,6 @@ def create_session(db_url, debug=False, pool_recycle=3600): return scopedsession -@contextlib.contextmanager -def session_manager(db_url, debug=False, pool_recycle=3600): - """ A handy context manager for our sessions. """ - session = yield from create_session( - db_url, debug=debug, pool_recycle=pool_recycle) - try: - yield session - session.commit() - except: - session.rollback() - raise - finally: - session.close() - - -@asyncio.coroutine def get_package(session, pkg_name): ''' Return information about a package, if we can find it. ''' @@ -98,17 +79,16 @@ def get_package(session, pkg_name): raise else: time.sleep(0.1) - output = yield from get_package(session, pkg_name) + output = get_package(session, pkg_name) return output -@asyncio.coroutine def get_package_by_src(session, pkg_name): ''' Return information about a package, if we can find it. ''' # First try if there is a package matching exactly the provided name - simple_match = yield from get_package(session, pkg_name) + simple_match = get_package(session, pkg_name) if simple_match and simple_match.basename == pkg_name: return simple_match @@ -133,10 +113,9 @@ def get_package_by_src(session, pkg_name): raise else: time.sleep(0.1) - yield from get_package_by_src(session, pkg_name) + get_package_by_src(session, pkg_name) -@asyncio.coroutine def get_package_by(session, tablename, key, cnt=None): ''' Return information the package providing the provides, if we can find it. ''' @@ -163,13 +142,12 @@ def get_package_by(session, tablename, key, cnt=None): raise else: time.sleep(0.1) - output = yield from get_package_by( + output = get_package_by( session, tablename, key, cnt=cnt) return output -@asyncio.coroutine def get_package_info(session, pkgKey, tablename): ''' Return the information contained in the specified table for the given package. @@ -192,12 +170,11 @@ def get_package_info(session, pkgKey, tablename): raise else: time.sleep(0.1) - output = yield from get_package_info(session, pkgKey, tablename) + output = get_package_info(session, pkgKey, tablename) return output -@asyncio.coroutine def get_co_packages(session, srcpkg_name): ''' Return the name of all the packages coming from the same source-package. @@ -217,12 +194,11 @@ def get_co_packages(session, srcpkg_name): raise else: time.sleep(0.1) - output = yield from get_co_packages(session, srcpkg_name) + output = get_co_packages(session, srcpkg_name) return output -@asyncio.coroutine def get_files(session, pkg_id): ''' Return the list of all the files in a package given its key. ''' @@ -244,12 +220,11 @@ def get_files(session, pkg_id): raise else: time.sleep(0.1) - output = yield from get_files(session, pkg_id) + output = get_files(session, pkg_id) return output -@asyncio.coroutine def get_changelog(session, pkg_id): ''' Return the list of all the changelog in a package given its key. ''' @@ -271,6 +246,6 @@ def get_changelog(session, pkg_id): raise else: time.sleep(0.1) - output = yield from get_changelog(session, pkg_id) + output = get_changelog(session, pkg_id) return output diff --git a/mdapi/server.py b/mdapi/server.py deleted file mode 100644 index bdc9ca1..0000000 --- a/mdapi/server.py +++ /dev/null @@ -1,20 +0,0 @@ -import logging -import logging.config - -from aiohttp import web - -from mdapi import CONFIG, _set_routes - - -def main(): - - logging.basicConfig() - logging.config.dictConfig(CONFIG.get("LOGGING") or {"version": 1}) - - app = web.Application() - app = _set_routes(app) - - host = CONFIG.get("HOST", "127.0.0.1") - port = CONFIG.get("PORT", 8080) - - web.run_app(app, host=host, port=port) diff --git a/requirements.txt b/requirements.txt index 345bfbf..2716f59 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,5 @@ -aiohttp >= 3.5.4 +flask fedora_messaging -# this is a requirement of aiohttp but better safe than sorry requests sqlalchemy -werkzeug -flufl.lock +gunicorn diff --git a/tests/test_mdapi_data.py b/tests/test_mdapi_data.py index 0922061..d221d0d 100644 --- a/tests/test_mdapi_data.py +++ b/tests/test_mdapi_data.py @@ -27,24 +27,18 @@ Tests for mdapi. ''' import json import os -import shutil import subprocess import sys -import tempfile -import mock + import pytest -from aiohttp import web -from sqlalchemy.exc import SQLAlchemyError +from mdapi import app sys.path.insert(0, os.path.join(os.path.dirname( os.path.abspath(__file__)), '..')) -import mdapi - HERE = os.path.join(os.path.dirname(os.path.abspath(__file__))) -TMPDIR = None @pytest.fixture(scope="module") def set_env(request): @@ -52,43 +46,33 @@ def set_env(request): Collects the sqlite database from the mirror to have some data to test against. """ - global TMPDIR - TMPDIR = tempfile.mkdtemp(prefix="mdapi-test-") - print("Creating %s" % TMPDIR) - configfile = os.path.join(TMPDIR, "config") + + tmpdir = "/tmp" + configfile = os.path.join(tmpdir, "config") with open(configfile, "w") as stream: - stream.write("DB_FOLDER = '%s'\n" % TMPDIR) + stream.write("DB_FOLDER = '%s'\n" % tmpdir) print("Downloading the databases...") subprocess.check_output( ["../mdapi-get_repo_md", configfile], cwd=HERE, ) - assert len(os.listdir(TMPDIR)) > 2 - - def clean_up(): - print("\nRemoving %s" % TMPDIR) - shutil.rmtree(TMPDIR) - request.addfinalizer(clean_up) + assert len(os.listdir(tmpdir)) > 2 @pytest.fixture -def tmpdir(): - return TMPDIR - +def client(set_env, tmpdir): + app.config['TESTING'] = True + app.config['DB_FOLDER'] = "/tmp" -@pytest.fixture -def cli(set_env, tmpdir, loop, aiohttp_client): - mdapi.CONFIG['DB_FOLDER'] = tmpdir - app = web.Application() - app = mdapi._set_routes(app) - return loop.run_until_complete(aiohttp_client(app)) + with app.test_client() as client: + yield client -async def test_view_index_page(cli): - resp = await cli.get('/') - assert resp.status == 200 +def test_view_index_page(client): + resp = client.get('/') + assert resp.status_code == 200 header = r""" _ _ | | (_) @@ -99,51 +83,47 @@ async def test_view_index_page(cli): | | |_| """ - output = await resp.text() - assert header in output + assert header in resp.get_data(as_text=True) -async def test_view_branches(cli): - resp = await cli.get('/branches') - assert resp.status == 200 - output = await resp.text() +def test_view_branches(client): + resp = client.get('/branches') + assert resp.status_code == 200 + output = resp.get_data(as_text=True) assert 'src_rawhide' in output assert 'rawhide' in output -async def test_view_pkg_rawhide(cli): - resp = await cli.get('/rawhide/pkg/kernel') - assert resp.status == 200 - json.loads(await resp.text()) +def test_view_pkg_rawhide(client): + resp = client.get('/rawhide/pkg/kernel') + assert resp.status_code == 200 -async def test_view_pkg_rawhide_invalid(cli): - resp = await cli.get('/rawhide/pkg/invalidpackagename') - assert resp.status == 404 - assert '404: Not Found' == await resp.text() +def test_view_pkg_rawhide_invalid(client): + resp = client.get('/rawhide/pkg/invalidpackagename') + assert resp.status_code == 404 -async def test_view_srcpkg_rawhide(cli): - resp = await cli.get('/rawhide/srcpkg/python-natsort') - assert resp.status == 200 - json.loads(await resp.text()) +def test_view_srcpkg_rawhide(client): + resp = client.get('/rawhide/srcpkg/python-natsort') + assert resp.status_code == 200 + json.loads(resp.get_data(as_text=True)) -async def test_view_file_list_rawhide(cli): - resp = await cli.get('/rawhide/files/kernel-core') - assert resp.status == 200 - json.loads(await resp.text()) +def test_view_file_list_rawhide(client): + resp = client.get('/rawhide/files/kernel-core') + assert resp.status_code == 200 + json.loads(resp.get_data(as_text=True)) -async def test_view_changelog_rawhide(cli): - resp = await cli.get('/rawhide/changelog/kernel') - assert resp.status == 200 - json.loads(await resp.text()) +def test_view_changelog_rawhide(client): + resp = client.get('/rawhide/changelog/kernel') + assert resp.status_code == 200 + json.loads(resp.get_data(as_text=True)) @pytest.mark.parametrize("action, package, status_code", [ ("requires", "R", 200), - ("provides", "perl(SetupLog)", 200), ("provides", "R", 200), ("obsoletes", "cabal2spec", 200), ("conflicts", "mariadb", 200), @@ -152,8 +132,8 @@ async def test_view_changelog_rawhide(cli): ("suggests", "httpd", 200), ("supplements", "(hunspell and langpacks-fr)", 200), ]) -async def test_view_property_koji(cli, action, package, status_code): - resp = await cli.get('/koji/%s/%s' % (action, package)) - assert resp.status == status_code +def test_view_property_koji(client, action, package, status_code): + resp = client.get('/koji/%s/%s' % (action, package)) + assert resp.status_code == status_code if status_code == 200: - json.loads(await resp.text()) + json.loads(resp.get_data(as_text=True)) diff --git a/tests/test_mdapi_empty.py b/tests/test_mdapi_empty.py index 484cb8b..07b3ac6 100644 --- a/tests/test_mdapi_empty.py +++ b/tests/test_mdapi_empty.py @@ -26,34 +26,29 @@ Tests for mdapi. ''' import os -import shutil -import subprocess import sys -import tempfile import pytest -from aiohttp import web +from mdapi import app sys.path.insert(0, os.path.join(os.path.dirname( os.path.abspath(__file__)), '..')) -import mdapi -import mdapi.lib - HERE = os.path.join(os.path.dirname(os.path.abspath(__file__))) @pytest.fixture -def cli(loop, aiohttp_client): - mdapi.CONFIG['DB_FOLDER'] = '.' - app = web.Application() - app = mdapi._set_routes(app) - return loop.run_until_complete(aiohttp_client(app)) +def client(): + app.config['TESTING'] = True + app.config['DB_FOLDER'] = "" + + with app.test_client() as client: + yield client -async def test_view_index_page(cli): - resp = await cli.get('/') - assert resp.status == 200 +def test_view_index_page(client): + resp = client.get('/') + assert resp.status_code == 200 header = r""" _ _ | | (_) @@ -64,51 +59,45 @@ async def test_view_index_page(cli): | | |_| """ - output = await resp.text() - assert header in output + assert header in resp.get_data(as_text=True) -async def test_view_branches(cli): - resp = await cli.get('/branches') - assert resp.status == 200 - assert '[]' == await resp.text() +def test_view_branches(client): + app.config['DB_FOLDER'] = "/" + resp = client.get('/branches') + assert resp.status_code == 200 + assert '[]\n' == resp.get_data(as_text=True) -async def test_view_pkg_rawhide(cli): - resp = await cli.get('/rawhide/pkg/kernel') - assert resp.status == 400 - assert '400: Bad Request' == await resp.text() +def test_view_pkg_rawhide(client): + resp = client.get('/rawhide/pkg/kernel') + assert resp.status_code == 400 -async def test_view_pkg_rawhide_invalid(cli): - resp = await cli.get('/rawhide/pkg/invalidpackagename') - assert resp.status == 400 - assert '400: Bad Request' == await resp.text() +def test_view_pkg_rawhide_invalid(client): + resp = client.get('/rawhide/pkg/invalidpackagename') + assert resp.status_code == 400 -async def test_view_srcpkg_rawhide(cli): - resp = await cli.get('/rawhide/srcpkg/python-natsort') - assert resp.status == 400 - assert '400: Bad Request' == await resp.text() +def test_view_srcpkg_rawhide(client): + resp = client.get('/rawhide/srcpkg/python-natsort') + assert resp.status_code == 400 -async def test_view_file_list_rawhide(cli): - resp = await cli.get('/rawhide/files/kernel-core') - assert resp.status == 400 - assert '400: Bad Request' == await resp.text() +def test_view_file_list_rawhide(client): + resp = client.get('/rawhide/files/kernel-core') + assert resp.status_code == 400 -async def test_view_changelog_rawhide(cli): - resp = await cli.get('/rawhide/changelog/kernel') - assert resp.status == 400 - assert '400: Bad Request' == await resp.text() +def test_view_changelog_rawhide(client): + resp = client.get('/rawhide/changelog/kernel') + assert resp.status_code == 400 @pytest.mark.parametrize("action", [ "requires", "provides", "obsoletes", "conflicts", "enhances", "recommends", "suggests", "supplements", ]) -async def test_view_property_koji(cli, action): - resp = await cli.get('/koji/%s/R' % action) - assert resp.status == 400 - assert '400: Bad Request' == await resp.text() +def test_view_property_koji(client, action): + resp = client.get('/koji/%s/R' % action) + assert resp.status_code == 400