From a0ce2e323b47b503e0347d69e02ecda6eb3881c7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 25 2016 11:24:59 +0000 Subject: [PATCH 1/7] Create the get_package_by method in the internal library This method can be used to query the requires, provides, obsoletes tables and retrieve the corresponding package. --- diff --git a/mdapi/lib.py b/mdapi/lib.py index 4d14d7c..bc0b79f 100644 --- a/mdapi/lib.py +++ b/mdapi/lib.py @@ -98,6 +98,36 @@ def get_package(session, pkg_name): return output +def get_package_by(session, tablename, key): + ''' Return information the package providing the provides, if we can find it. + ''' + table = getattr(primary, tablename.capitalize()) + + cnt = 0 + try: + pkg = session.query( + primary.Package + ).filter( + table.name == key + ).filter( + table.pkgKey == primary.Package.pkgKey + ).order_by( + primary.Package.epoch.desc(), + primary.Package.version.desc(), + primary.Package.release.desc(), + ) + output = pkg.all() + except SQLAlchemyError as err: + cnt += 1 + if cnt > RETRY_ATTEMPT: + raise + else: + time.sleep(0.1) + output = get_package_by_provides(session, pkg_name) + + return output + + def get_package_info(session, pkgKey, tablename): ''' Return the information contained in the specified table for the given package. From 227cbd1cb69aecdb8a0bc35b12349e9f13586dcd Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 25 2016 11:28:36 +0000 Subject: [PATCH 2/7] Create a dedicate method to retrieve informations about one or more packages --- diff --git a/mdapi/__init__.py b/mdapi/__init__.py index 473773b..26d7e2c 100644 --- a/mdapi/__init__.py +++ b/mdapi/__init__.py @@ -103,6 +103,59 @@ def _get_pretty(request): return pretty +def _expand_pkg_info(pkgs, branch, repotype=None): + ''' Return a JSON blob containing all the information we want to return + for the provided package or packages. + ''' + singleton = False + if not isinstance(pkgs, (list, tuple)): + singleton = True + pkgs = [pkgs] + output = [] + for pkg in pkgs: + out = pkg.to_json() + dbfile = '%s/mdapi-%s%s-primary.sqlite' % ( + CONFIG['DB_FOLDER'], branch, '-%s' % repotype if repotype else '') + + with file_lock.FileFlock(dbfile + '.lock'): + session = mdapilib.create_session('sqlite:///%s' % dbfile) + # Fill in some extra info + + # Basic infos, always present regardless of the version of the repo + for datatype in ['conflicts', 'obsoletes', 'provides', 'requires']: + data = mdapilib.get_package_info( + session, pkg.pkgKey, datatype.capitalize()) + if data: + out[datatype] = [item.to_json() for item in data] + else: + out[datatype] = data + + # New meta-data present for soft dependency management in RPM + for datatype in ['enhances', 'recommends', 'suggests', 'supplements']: + data = mdapilib.get_package_info( + session, pkg.pkgKey, datatype.capitalize()) + if data: + out[datatype] = [item.to_json() for item in data] + else: + out[datatype] = data + + # Add the list of packages built from the same src.rpm + if pkg.rpm_sourcerpm: + out['co-packages'] = list(set([ + cpkg.name + for cpkg in mdapilib.get_co_packages(session, pkg.rpm_sourcerpm) + ])) + else: + out['co-packages'] = [] + out['repo'] = repotype if repotype else 'release' + session.close() + output.append(out) + if singleton: + return output[0] + else: + return output + + @asyncio.coroutine def get_pkg(request): branch = request.match_info.get('branch') @@ -110,43 +163,7 @@ def get_pkg(request): name = request.match_info.get('name') pkg, repotype = _get_pkg(branch, name) - output = pkg.to_json() - - dbfile = '%s/mdapi-%s%s-primary.sqlite' % ( - CONFIG['DB_FOLDER'], branch, '-%s' % repotype if repotype else '') - - with file_lock.FileFlock(dbfile + '.lock'): - session = mdapilib.create_session('sqlite:///%s' % dbfile) - # Fill in some extra info - - # Basic infos, always present regardless of the version of the repo - for datatype in ['conflicts', 'obsoletes', 'provides', 'requires']: - data = mdapilib.get_package_info( - session, pkg.pkgKey, datatype.capitalize()) - if data: - output[datatype] = [item.to_json() for item in data] - else: - output[datatype] = data - - # New meta-data present for soft dependency management in RPM - for datatype in ['enhances', 'recommends', 'suggests', 'supplements']: - data = mdapilib.get_package_info( - session, pkg.pkgKey, datatype.capitalize()) - if data: - output[datatype] = [item.to_json() for item in data] - else: - output[datatype] = data - - # Add the list of packages built from the same src.rpm - if pkg.rpm_sourcerpm: - output['co-packages'] = list(set([ - cpkg.name - for cpkg in mdapilib.get_co_packages(session, pkg.rpm_sourcerpm) - ])) - else: - output['co-packages'] = [] - output['repo'] = repotype if repotype else 'release' - session.close() + output = _expand_pkg_info(pkg, branch, repotype) args = {} if pretty: From 7488045696ddb97564dce09ebb29cb27b0b21415 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 25 2016 11:29:14 +0000 Subject: [PATCH 3/7] Create the requires, provides and obsoletes endpoints This endpoints returns the package having the specified properties, they provide, require or obsolete the specified component. --- diff --git a/mdapi/__init__.py b/mdapi/__init__.py index 26d7e2c..5369c5a 100644 --- a/mdapi/__init__.py +++ b/mdapi/__init__.py @@ -55,7 +55,7 @@ with open(indexfile) as stream: INDEX = INDEX.replace('$PREFIX', CONFIG.get('PREFIX', '')) -def _get_pkg(branch, name): +def _get_pkg(branch, name, action=None): ''' Return the pkg information for the given package in the specified branch or raise an aiohttp exception. ''' @@ -77,7 +77,10 @@ def _get_pkg(branch, name): with file_lock.FileFlock(dbfile + '.lock'): session = mdapilib.create_session('sqlite:///%s' % dbfile) - pkg = mdapilib.get_package(session, name) + if action: + pkg = mdapilib.get_package_by(session, action, name) + else: + pkg = mdapilib.get_package(session, name) session.close() if pkg: break @@ -247,6 +250,40 @@ def list_branches(request): return web.Response(body=json.dumps(output, **args).encode('utf-8')) +def process_dep(request, action): + ''' Return the information about the packages having the specified + action (provides, requires, obsoletes...) + ''' + branch = request.match_info.get('branch') + pretty = _get_pretty(request) + name = request.match_info.get('name') + + pkg, repotype = _get_pkg(branch, name, action=action) + + output = _expand_pkg_info(pkg, branch, repotype) + + args = {} + if pretty: + args = dict(sort_keys=True, indent=4, separators=(',', ': ')) + + return web.Response(body=json.dumps(output, **args).encode('utf-8')) + + +@asyncio.coroutine +def get_provides(request): + return process_dep(request, 'provides') + + +@asyncio.coroutine +def get_requires(request): + return process_dep(request, 'requires') + + +@asyncio.coroutine +def get_obsoletes(request): + return process_dep(request, 'obsoletes') + + @asyncio.coroutine def index(request): return web.Response(body=INDEX.encode('utf-8')) @@ -264,6 +301,9 @@ def init(loop): ('/', index), ('/branches', list_branches), ('/{branch}/pkg/{name}', get_pkg), + ('/{branch}/provides/{name}', get_provides), + ('/{branch}/requires/{name}', get_requires), + ('/{branch}/obsoletes/{name}', get_obsoletes), ('/{branch}/files/{name}', get_pkg_files), ('/{branch}/changelog/{name}', get_pkg_changelog), ]) From ad547734af4eafedb12d723fa3d053364d050da7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 25 2016 11:44:41 +0000 Subject: [PATCH 4/7] Add the endpoints for the soft-dependencies --- diff --git a/mdapi/__init__.py b/mdapi/__init__.py index 5369c5a..1b0abf4 100644 --- a/mdapi/__init__.py +++ b/mdapi/__init__.py @@ -285,6 +285,31 @@ def get_obsoletes(request): @asyncio.coroutine +def get_conflicts(request): + return process_dep(request, 'conflicts') + + +@asyncio.coroutine +def get_enhances(request): + return process_dep(request, 'enhances') + + +@asyncio.coroutine +def get_recommends(request): + return process_dep(request, 'recommends') + + +@asyncio.coroutine +def get_suggests(request): + return process_dep(request, 'suggests') + + +@asyncio.coroutine +def get_supplements(request): + return process_dep(request, 'supplements') + + +@asyncio.coroutine def index(request): return web.Response(body=INDEX.encode('utf-8')) @@ -301,9 +326,17 @@ def init(loop): ('/', index), ('/branches', list_branches), ('/{branch}/pkg/{name}', get_pkg), + ('/{branch}/provides/{name}', get_provides), ('/{branch}/requires/{name}', get_requires), ('/{branch}/obsoletes/{name}', get_obsoletes), + ('/{branch}/conflicts/{name}', get_conflicts), + + ('/{branch}/enhances/{name}', get_enhances), + ('/{branch}/recommends/{name}', get_recommends), + ('/{branch}/suggests/{name}', get_suggests), + ('/{branch}/supplements/{name}', get_supplements), + ('/{branch}/files/{name}', get_pkg_files), ('/{branch}/changelog/{name}', get_pkg_changelog), ]) From a83aafaa1b5a16e04906598c287cf0e1f2339191 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 25 2016 11:44:55 +0000 Subject: [PATCH 5/7] Adjust the documentation in the front page --- diff --git a/mdapi/index.html b/mdapi/index.html index 9053b57..b938ee8 100644 --- a/mdapi/index.html +++ b/mdapi/index.html @@ -109,6 +109,47 @@ So for example, for the kernel in rawhide: /rawhide/changelog/kernel +Retrieve the packages having a specific property +------------------------------------------------ + +You can retrieve the list of packages having a specific property. These +properties can be any of: + requires, provides, obsoletes, conflicts, enhances, recommends, suggests, supplements. + + /{property}/{package name} + +For example to retrieve the list of packages that require a specific package: + + /requires/{package name} + +Few examples: + + packages requiring R in rawhide: + /rawhide/requires/R + + packages providing perl(SetupLog) in rawhide: + /rawhide/provides/perl(SetupLog) + + packages obsoleting cabal2spec in rawhide: + rawhide/obsoletes/cabal2spec + + packages conflicting with mariadb in rawhide: + rawhide/conflicts/mariadb + + packages enhancing vagrant in rawhide: + rawhide/enhances/vagrant + + packages recommending python3-dateutils in rawhide: + rawhide/recommends/python3-dateutils + + packages suggesting tag in rawhide: + rawhide/suggests/tar + + packages supplementing `(hunspell and langpacks-fr)` in rawhide: + rawhide/supplements/(hunspell and langpacks-fr) + + + |‾| From 3da2c5596437ddedcdad7227f24c822e98e1e9d4 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 25 2016 13:52:12 +0000 Subject: [PATCH 6/7] Fix the recursion so that we don't end up in an infinite loop --- diff --git a/mdapi/lib.py b/mdapi/lib.py index bc0b79f..67d7f3b 100644 --- a/mdapi/lib.py +++ b/mdapi/lib.py @@ -98,12 +98,12 @@ def get_package(session, pkg_name): return output -def get_package_by(session, tablename, key): +def get_package_by(session, tablename, key, cnt=None): ''' Return information the package providing the provides, if we can find it. ''' table = getattr(primary, tablename.capitalize()) - cnt = 0 + cnt = cnt or 0 try: pkg = session.query( primary.Package @@ -123,7 +123,7 @@ def get_package_by(session, tablename, key): raise else: time.sleep(0.1) - output = get_package_by_provides(session, pkg_name) + output = get_package_by(session, tablename, key, cnt=cnt) return output From 6addcc37aaa8fbbc63eba2139757e57ed9c7a376 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 25 2016 13:52:31 +0000 Subject: [PATCH 7/7] Catch exception and raise an error 400 --- diff --git a/mdapi/__init__.py b/mdapi/__init__.py index 1b0abf4..88510ba 100644 --- a/mdapi/__init__.py +++ b/mdapi/__init__.py @@ -258,7 +258,10 @@ def process_dep(request, action): pretty = _get_pretty(request) name = request.match_info.get('name') - pkg, repotype = _get_pkg(branch, name, action=action) + try: + pkg, repotype = _get_pkg(branch, name, action=action) + except: + raise web.HTTPBadRequest() output = _expand_pkg_info(pkg, branch, repotype)