From f9eb7f77f871c49dca23227ace9f4fe5ca0e8d2a Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sep 26 2017 07:07:36 +0000 Subject: Rename CLI & Python module to fedmod - modularity & modtools are both overly generic - modulatiry & modtools are both claimed on PyPI - fedmod is short for "Fedora Modularity" - fedmod used a similar naming style to fedmsg and fedpkg - fedmod is available on PyPI Also cleans up an issue where the packaged script would have conflicted with the generated entry point by instead making the package executable with Python's `-m` switch --- diff --git a/fedmod/__init__.py b/fedmod/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/fedmod/__init__.py diff --git a/fedmod/__main__.py b/fedmod/__main__.py new file mode 100644 index 0000000..9e88f92 --- /dev/null +++ b/fedmod/__main__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 + +import sys +from .cli import ModtoolsCLIHelper + + +if __name__ == "__main__": + cli = ModtoolsCLIHelper() + sys.exit(cli.run()) diff --git a/fedmod/cli.py b/fedmod/cli.py new file mode 100644 index 0000000..fb5c333 --- /dev/null +++ b/fedmod/cli.py @@ -0,0 +1,115 @@ +import sys +import argparse +import logging + +from .module_generator import ModuleGenerator +from .oc_template import OpenShiftTemplateGenerator +from .mod2dockerfile import ModulemdDockerfileGenerator + + +class ModtoolsCLI(object): + """ Class for processing data from commandline """ + + @staticmethod + def build_parser(): + parser = argparse.ArgumentParser(description="Generates module related files") + base_parser = argparse.ArgumentParser(add_help=False) + base_parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="verbose operation" + ) + + subparsers = parser.add_subparsers(dest="cmd_name") + + parser_rpm2module = subparsers.add_parser( + 'rpm2module', parents=[base_parser], + help="Generates modulemd file", + description="Gets package info and dependencies and creates modulemd file." + ) + parser_rpm2module.add_argument( + "pkgs", + metavar='PKGS', + nargs='+', + help="Specify list of packages for module.", + ) + + parser_docker2openshift = subparsers.add_parser( + "docker2openshift", parents=[base_parser], + help="Generates openshift template from dockerfile", + description="Creates an OpenShift template YAML file.", + ) + + parser_docker2openshift.add_argument( + "image", + metavar='IMAGE', + help="docker image name (like NAME or docker.io/USER/NAME)", + ) + parser_docker2openshift.add_argument( + "--dockerfile", + help="Specify Dockerfile name. Default is Dockerfile." + ) + + parser_module2dockerfile = subparsers.add_parser( + 'module2dockerfile', parents=[base_parser], + help="Generates dockerfile from modulemd file", + description="Creates Dockerfile with suggestions and pre-filled values" + ) + + parser_module2dockerfile.add_argument( + "modulemd_file", + metavar='MODULEMD_FILE', + help='Specify path to modulemd file' + ) + + parser_module2dockerfile.add_argument( + "--output", + nargs='?', + help='Specify custom output file.' + ) + + parser_module2dockerfile.add_argument( + "--template", + help='Specify custom Dockerfile template', + required=True + ) + + return parser + + def __init__(self, args=None): + self.parser = ModtoolsCLI.build_parser() + self.args = self.parser.parse_args(args) + + def __getattr__(self, name): + try: + return getattr(self.args, name) + except AttributeError: + return object.__getattribute__(self, name) + + +class ModtoolsCLIHelper(object): + + @staticmethod + def run(): + try: + cli = ModtoolsCLI(sys.argv[1:]) + if cli.args.verbose: + logging.basicConfig(level=logging.INFO) + if cli.args.cmd_name == 'rpm2module': + mg = ModuleGenerator(cli.args.pkgs) + mg.run() + + if cli.args.cmd_name == 'docker2openshift': + otg = OpenShiftTemplateGenerator(cli.args) + otg.run() + + if cli.args.cmd_name == 'module2dockerfile': + mtd = ModulemdDockerfileGenerator(cli.args) + mtd.run() + + except KeyboardInterrupt: + print('\nInterrupted by user') + except Exception as e: + print(e) + sys.exit(1) diff --git a/fedmod/mod2dockerfile.py b/fedmod/mod2dockerfile.py new file mode 100644 index 0000000..dafee67 --- /dev/null +++ b/fedmod/mod2dockerfile.py @@ -0,0 +1,49 @@ +import modulemd +import os +import tempfile +import shutil +from string import Template + + +class PercentTemplate(Template): + delimiter = '%%' + + +class ModulemdDockerfileGenerator(object): + + def __init__(self, args=None): + self.dockerfile_path = args.output + self.modulemd_path = args.modulemd_file + self.template_path=args.template + + def load_modulemd(self): + self.mmd = modulemd.ModuleMetadata() + self.mmd.load(self.modulemd_path) + + self.values = {'NAME': self.mmd.name,'VERSION': self.mmd.version, + 'DESCRIPTION': self.mmd.description, 'SUMMARY': self.mmd.summary, + 'API_PACKAGES': ' '.join(self.mmd.api.rpms)} + + def generate_dockerfile(self): + tmplFile = open(self.template_path) + tmpl = PercentTemplate(tmplFile.read()) + self.result = tmpl.safe_substitute(self.values) + + def save_result(self): + if self.dockerfile_path is not None: + output_file = self.dockerfile_path + else: + tmp_dir = tempfile.mkdtemp() + if os.path.isdir(tmp_dir): + shutil.rmtree(tmp_dir) + os.makedirs(tmp_dir) + output_file = os.path.join(tmp_dir, os.path.basename('Dockerfile')) + + with open(output_file, 'w') as f: + f.write(self.result) + print("Modulemd template is generated here: %s" % (output_file)) + + def run(self): + self.load_modulemd() + self.generate_dockerfile() + self.save_result() diff --git a/fedmod/module_deps_differ.py b/fedmod/module_deps_differ.py new file mode 100755 index 0000000..4403b81 --- /dev/null +++ b/fedmod/module_deps_differ.py @@ -0,0 +1,237 @@ +import os +import logging + +import dnf + +from distutils.version import LooseVersion +from pdc_client import PDCClient + + +FALLBACK_STREAM = 'master' +STREAM = 'f26' +REPO_F26 = "http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/x86_64/os/" +REPO_F26_SOURCE = "http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/source/tree/" +ARCH = 'x86_64' + +# TODO: add ability to solve more packages in one run +# TODO: solve architecture! - as option? + + +p = None + + +def get_pkgs_source_rpm_name(pkg): + """ + pkg: dnf.Package + + returns name of source RPM used to build the provided rpm + """ + if LooseVersion(dnf.__version__) < LooseVersion("2.0.0"): + source_rpm = pkg.sourcerpm + source_rpm_name = source_rpm.rsplit('-', 2)[0] + else: + source_rpm_name = pkg.source_name + return source_rpm_name + + +def get_pdc_client(): + """ cache PDCClient instance """ + global p + if p is None: + p = PDCClient('https://pdc.fedoraproject.org/rest_api/v1/', develop=True, ssl_verify=True) + return p + + +class ModuleDepsDiffer(object): + + def __init__(self, pkgs): + self.p = get_pdc_client() + self.repos = dict() + self.repo_provides = dict() + # TODO: get this from args + self.packages = list(pkgs) + self.build_req = set() + self.runtime_req = set() + self.build_caps_classified = dict() + self.build_caps_classified['result'] = set() + self.runtime_caps_classified = dict() + self.runtime_caps_classified['result'] = set() + self.repo_bases = {} + self.module_id_to_name = {} + + def get_module_name(self, koji_tag_name): + return self.module_id_to_name.get(koji_tag_name, koji_tag_name) + + def obtain_module_names(self): + logging.info('Scanning built modules, this might take a while.') + j = self.p['unreleasedvariants']( + variant_type="module", + active=True, + page_size=-1 + # variant_version=STREAM, + ) + logging.info('Scanning built modules - DONE.') + for module in j: + self.module_id_to_name[module["koji_tag"]] = "{}:{}".format(module["variant_id"], module["variant_version"]) + + @staticmethod + def get_base_from_repo(reponame, repourl): + logging.info('Loading repo: %s', reponame) + base = dnf.Base() + if LooseVersion(dnf.__version__) < LooseVersion("2.0.0"): + repo = dnf.repo.Repo(reponame, base.conf.cachedir) + else: + repo = dnf.repo.Repo(reponame, base.conf) + repo.baseurl = repourl + repo.load() + repo.enable() + base.repos.add(repo) + base.fill_sack(load_available_repos=True, load_system_repo=False) + return base + + def _init_repo_bases(self): + """ + version 1 is no longer usable since koji no longer creates repos for + modules this version iterates over repos present in + ~/modulebuild/cache/koji_tags, this means that you should do `mbs-build + local` to populate the directory + + Once we have real compose of boltron, we might utilize it here + + initialize repo objects for dnf to kick off queries + """ + logging.warn("Getting module information from mbs cache." + + " Please run mbs-build local with dependency modules" + + " you are interested in before running this script.") + cache_path = os.path.expanduser("~/modulebuild/cache/koji_tags") + for module_name in os.listdir(cache_path): + # for now ignore all what is not in f26 stream, remove when situation changes + if not self.module_id_to_name[module_name].endswith(':f26'): + continue + if module_name != "f26-modularity": # bootstrap contains almost everything, so let's ignore it + self.repos[module_name] = "file://{}".format(os.path.join(cache_path, module_name)) + for reponame, repourl in self.repos.items(): + base = ModuleDepsDiffer.get_base_from_repo(reponame, repourl) + self.repo_bases[reponame] = base + + def get_package_requires(self): + # runtime + logging.info('Getting package requirements') + pkgs = [] + base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26) + for pkg in self.packages: + filter_result = base.sack.query().filter(name=pkg, arch=['noarch', ARCH], latest=True) + ModuleDepsDiffer.package_unique(filter_result) + pkgs.append(filter_result[0]) + for pkg in pkgs: + requires = getattr(pkg, 'requires') + for q in requires: + self.runtime_req.add(str(q)) + + # build + source_pkgs = [] + base = ModuleDepsDiffer.get_base_from_repo('f26-source', REPO_F26_SOURCE) + for pkg in pkgs: + source_rpm_name = get_pkgs_source_rpm_name(pkg) + filter_result = base.sack.query().filter(name=source_rpm_name) + ModuleDepsDiffer.package_unique(filter_result) + source_pkgs.append(filter_result[0]) + for pkg in source_pkgs: + requires = getattr(pkg, 'requires') + for q in requires: + self.build_req.add(str(q)) + + # Igor suggests to do complete dependency solving here: + # base.install('package'), installroot=, make sure the right repos are enabled + # base.resolve() + # and then analyze base.transaction.install_set + # weak deps will get resolved correctly with this approach + def classify_caps(self, caps_classified, requirements): + for cap in requirements: + found = False + for reponame, base in self.repo_bases.items(): + q = base.sack.query() + pkg = q.filter(provides=[cap], arch=['noarch', ARCH], latest=True) + if pkg: + found = True + caps_classified.setdefault(reponame, set()) + caps_classified[reponame].add(cap) + # don't break here in case one cap is present in multiple repos + if not found: + caps_classified['result'].add(cap) + + @staticmethod + def package_unique(result): + if len(result) > 1: + raise ValueError('Name of package is not unique: ' + str(result.result)) + if len(result) == 0: + raise ValueError('No package found in repo') + + @staticmethod + def whatprovides(caps): + result = set() + base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26) + q = base.sack.query() + for cap in caps: + logging.info('Getting source package for %s:', cap) + pkg = q.filter(provides__glob=[cap], arch=['noarch', ARCH], latest=True) + if not pkg: + pkg = q.filter(file__glob=cap) + if len(pkg) == 0: + raise ValueError('No package provides capability \'' + cap + '\'') + source_rpm_name = get_pkgs_source_rpm_name(pkg.result[0]) + result.add(source_rpm_name) + return result + + def _dump_caps_result(self, items): + for repo, caps in items: + if not caps: + continue + if repo == 'result': + print("Capabilities which aren't provided by any module:") + else: + print("Components which are part of " + + self.get_module_name(repo) + + " module:") + for cap in sorted(caps): + print(cap) + print('') + + def dump_caps_result(self): + print('BUILD requirements:') + print('-------------------') + self._dump_caps_result(self.build_caps_classified.items()) + print('\n') + print('RUNTIME requirements:') + print('---------------------') + self._dump_caps_result(self.runtime_caps_classified.items()) + + def _dump_pkgs_result(self, items): + for repo, caps in items: + if not caps: + continue + if repo == 'result': + print("Components which aren't provided by any module:") + else: + print("Components which are part of " + + self.get_module_name(repo) + + " module:") + for cap in sorted(ModuleDepsDiffer.whatprovides(caps)): + print(cap) + print('') + + def dump_pkgs_result(self): + print('BUILD dependencies:') + print('-------------------') + self._dump_pkgs_result(self.build_caps_classified.items()) + print('\n') + print('RUNTIME dependencies:') + print('---------------------') + self._dump_pkgs_result(self.runtime_caps_classified.items()) + + def run(self): + self.obtain_module_names() + self._init_repo_bases() + self.get_package_requires() + self.classify_caps(self.build_caps_classified, self.build_req) + self.classify_caps(self.runtime_caps_classified, self.runtime_req) diff --git a/fedmod/module_generator.py b/fedmod/module_generator.py new file mode 100644 index 0000000..c8932ce --- /dev/null +++ b/fedmod/module_generator.py @@ -0,0 +1,120 @@ +from __future__ import absolute_import + +import modulemd +import dnf +from .module_deps_differ import ModuleDepsDiffer +import logging + +class ModuleGenerator(object): + + def __init__(self, pkgs): + self.pkgs = pkgs + self.pkg = None + self.mmd = modulemd.ModuleMetadata() + self.build_deps = set() + self.run_deps = set() + self.differ = ModuleDepsDiffer(pkgs) + + def _save_module_md(self): + """ + Function saves modulemd file to the current directory + based on argument name + :return: + """ + + if len(self.pkgs) == 1: + file_name = self.pkgs[0] + '.yaml' + else: + file_name = "modulemd-output.yaml" + self.mmd.dump(file_name) + print('Modulemd file is generated here ./%s' % file_name) + return True + + def _update_module_md(self): + """ + Function updates modulemd file with dependencies + are information taken from SPEC file. + :return: + """ + self.mmd.add_module_license("MIT") + + if len(self.pkgs) == 1: + self.mmd.summary = str(self.pkg.summary) + self.mmd.description = str(self.pkg.description) + + # Default license for the module metadata, same as default Fedora + # content license. + + self.mmd.add_content_license(str(self.pkg.license)) + + for pkg in self.pkgs: + self.mmd.api.add_rpm(pkg) + self.mmd.components.add_rpm(ModuleDepsDiffer.whatprovides([pkg]).pop(), "Package in api", buildorder=self._get_build_order(pkg)) + + for pkg in (self.build_deps - self.mmd.api.rpms - self.run_deps): + self.mmd.filter.add_rpm(pkg) + + for pkg in self.build_deps.intersection(self.run_deps): + self.mmd.components.add_rpm(pkg, "Build and runtime dependency.", buildorder=self._get_build_order(pkg)) + + for pkg in (self.build_deps - self.run_deps): + self.mmd.components.add_rpm(pkg, "Build dependency.", buildorder=self._get_build_order(pkg)) + + for pkg in (self.run_deps - self.build_deps): + self.mmd.components.add_rpm(pkg, "Runtime dependency.", buildorder=self._get_build_order(pkg)) + + for mod, caps in self.differ.build_caps_classified.items(): + if mod == 'result': + continue + if caps: + name, stream = self.differ.get_module_name(mod).split(':') + self.mmd.add_buildrequires(name, stream) + + for mod, caps in self.differ.runtime_caps_classified.items(): + if mod == 'result': + continue + if caps: + name, stream = self.differ.get_module_name(mod).split(':') + self.mmd.add_requires(name, stream) + + def _get_build_order(self, pkg): + if pkg in self.mmd.api.rpms: + return 10 + else: + return 0 + + def _get_pkg_info(self): + """ + Function loads package from dnf + :return: + """ + logging.info("Getting package info from DNF") + b = dnf.Base() + b.read_all_repos() + b.fill_sack() + + q = b.sack.query().filter(name=self.pkgs, reponame='fedora', latest=True) + + if len(q) > 1: + raise ValueError('Name of package is not unique') + if len(q) == 0: + raise ValueError('No package found in repo') + self.pkg = q[0] + + def _get_dependencies(self): + """ + Function gets build and runtime dependencies of package + :return: + """ + logging.info('Dependency resolution started') + self.differ.run() + self.build_deps = ModuleDepsDiffer.whatprovides(self.differ.build_caps_classified['result']) + self.run_deps = ModuleDepsDiffer.whatprovides(self.differ.runtime_caps_classified['result']) + logging.info('Dependency resolution finished succesfully.') + + def run(self): + if len(self.pkgs) == 1: + self._get_pkg_info() + self._get_dependencies() + self._update_module_md() + self._save_module_md() diff --git a/fedmod/oc_template.py b/fedmod/oc_template.py new file mode 100644 index 0000000..10b803e --- /dev/null +++ b/fedmod/oc_template.py @@ -0,0 +1,335 @@ +from __future__ import absolute_import, print_function + +import os +import ast +import yaml +import tempfile +import shutil +import re +import shlex + +from dockerfile_parse import DockerfileParser + +# Dockerfile path +DOCKERFILE = "Dockerfile" + +EXPOSE = "EXPOSE" +VOLUME = "VOLUME" +LABEL = "LABEL" +ENV = "ENV" +PORTS = "PORTS" + +# OpenShift template +OPENSHIFT_TEMPLATE = "openshift-template.yml" + + +def get_string(value): + return ast.literal_eval(value) + + +class OpenShiftTemplateGenerator(object): + """ + Class generates an OpenShift template + It requires openshift-template.yml file. + """ + + dockerfile = None + oc_template = None + docker_dict = {} + + def __init__(self, args=None, dir_name=None): + if dir_name is None: + self.dir = os.getcwd() + else: + self.dir = dir_name + self.docker_image = args.image + if args.dockerfile is None: + self.dockerfile = 'Dockerfile' + else: + self.dockerfile = os.path.join(self.dir, args.dockerfile) + self.docker_dict = {} + + def _exist_docker_file(self): + """ + Function checks if docker file exists + :return: True if exists + """ + if not os.path.exists(self.dockerfile): + print("Dockerfile has to exists in the %s directory." % self.dir) + return False + return True + + def _exist_openshift_template(self): + """ + Function checks if openshift template exists + :return: True if exists + """ + if self.oc_template is None: + print("%s has to exists in the %s directory." % (OPENSHIFT_TEMPLATE, self.dir)) + return False + return True + + def _get_openshift_template(self): + """ + Function sets openshift template. + """ + for f in os.listdir(self.dir): + if os.path.isdir(os.path.join(self.dir, f)): + continue + file_name = os.path.join(self.dir, f) + if f == OPENSHIFT_TEMPLATE: + self.oc_template = file_name + + def _get_expose(self, value): + """Function returns exposes as field""" + return value.split() + + def _get_env(self, value): + """Function gets env as field""" + return shlex.split(value) + + def _get_volume(self, value): + """Function evaluates a value and returns as string.""" + return get_string(value) + + def _get_label(self, value): + """ + Function returns label from Docker file + except INSTALL, UNINSTALL and RUN label used by atomic. + :param value: row from Dockerfile + :return: label_dict + """ + untracked_values = ['INSTALL', 'UNINSTALL', 'RUN'] + if [f for f in untracked_values if value.startswith(f)]: + return None + labels = re.sub('\s\s+', ';', value).split(';') + labels = [l.replace('"', '') for l in labels] + label_dict = {} + for l in labels: + if len(l.split('=')) == 2: + label_dict[l.split('=')[0]] = l.split('=')[1] + elif re.match('maintainer', l, re.I): + label_dict['maintainer'] = l.split(' ',1)[1] + else: + raise ValueError("Unrecogised label: ", l) + return label_dict + + def _get_docker_tags(self): + """ + Function analyses dockerfile and extracts + ENV, VOLUME, EXPOSE and LABEL directives. + """ + if not self._exist_docker_file(): + return + tmp_dir = tempfile.mkdtemp() + if os.path.isdir(tmp_dir): + shutil.rmtree(tmp_dir) + os.makedirs(tmp_dir) + shutil.copyfile(self.dockerfile, os.path.join(tmp_dir, "Dockerfile")) + dfp = DockerfileParser(path=tmp_dir) + inst = "instruction" + allowed_tags = [ENV, EXPOSE, VOLUME, LABEL] + functions = {ENV: self._get_env, + EXPOSE: self._get_expose, + VOLUME: self._get_volume, + LABEL: self._get_label} + + for struct in dfp.structure: + key = struct[inst] + val = struct["value"] + if key in allowed_tags: + if key == LABEL: + if key not in self.docker_dict: + self.docker_dict[key] = {} + value = functions[key](val) + if value is not None: + self.docker_dict[key].update(value) + else: + if key not in self.docker_dict: + self.docker_dict[key] = [] + ret_val = functions[key](val) + for v in ret_val: + if v not in self.docker_dict[key]: + self.docker_dict[key].append(v) + + shutil.rmtree(tmp_dir) + + def _load_oc_template(self): + """ + Function loads openshift template + :return: YAML dictionary + """ + if not self._exist_openshift_template(): + return None + with open(self.oc_template, 'r') as f: + try: + templ = yaml.load(f) + except yaml.YAMLError as exc: + print(exc) + raise + return templ + + def _get_labels(self, templ): + labels = None + try: + labels = templ['metadata']['labels'] + except KeyError: + labels = {} + raise_exception = False + try: + labels['description'] = self.docker_dict[LABEL]['description'] + except KeyError: + print("Label Description is missing in Dockerfile. It is mandatory.") + raise_exception = True + try: + labels['tags'] = self.docker_dict[LABEL]['io.openshift.tags'] + except KeyError: + print('Label tags is missing in Dockerfile. It is mandatory.') + raise_exception = True + if raise_exception: + raise KeyError + labels['template'] = self.docker_image + return labels + + def _get_docker_labels(self): + """ + Function returns docker labels + :return: label dictionary + """ + if LABEL in self.docker_dict and self.docker_dict[LABEL]: + return self.docker_dict[LABEL] + return None + + def _get_docker_volumes(self): + """ + Function returns docker volumes and labels + :return: volume list, volume names + """ + volume_list = [] + volume_names = [] + if VOLUME in self.docker_dict and self.docker_dict[VOLUME]: + for p in self.docker_dict[VOLUME]: + volume_list.append({'mountPath': p, + 'name': 'name' + p.replace('/', '-')}) + volume_names.append({'name': 'name' + p.replace('/', '-'), + 'emptyDir': {} + }) + return volume_list, volume_names + + def _get_docker_env(self): + """ + Function return docker ENV directives + :return: list of ENV variables + """ + env_list = [] + if ENV in self.docker_dict and self.docker_dict[ENV]: + for e in self.docker_dict[ENV]: + key, val = e.split('=') + env_list.append({'name': key, + 'value': val}) + return env_list + + def _get_docker_expose(self): + """ + Function return docker EXPOSE directives + :return: list of PORTS + """ + ports_list = [] + if EXPOSE in self.docker_dict and self.docker_dict[EXPOSE]: + for p in self.docker_dict[EXPOSE]: + ports_list.append({'containerPort': int(p)}) + return ports_list + + def write_oc_template(self, templ): + """ + Function writes a YAML dictionary into template + :param templ: YAML template with all data + :return: + """ + tmp_dir = tempfile.mkdtemp() + if os.path.isdir(tmp_dir): + shutil.rmtree(tmp_dir) + os.makedirs(tmp_dir) + tmp_file = os.path.join(tmp_dir, os.path.basename(self.oc_template)) + with open(tmp_file, 'w') as f: + try: + yaml.safe_dump(templ, f, default_flow_style=False) + print("OpenShift template is generated here: %s" % (tmp_file)) + except yaml.YAMLError as exc: + print(exc) + raise + + def get_docker_directives(self, templ): + """ + Function collects all directives + :param templ: + :return: label_list, volume_list, volume_names, env_list, ports_list + """ + labels = volume_list = volume_names = env_list = ports_list = None + if self.docker_dict: + labels = self._get_labels(templ) + volume_list, volume_names = self._get_docker_volumes() + env_list = self._get_docker_env() + ports_list = self._get_docker_expose() + return labels, volume_list, volume_names, env_list, ports_list + + def generate_oc_template(self, templ, labels, volume_list, volume_names, env_list, ports_list): + """ + Function fulfills template with data taken from Dockerfile. + :param templ: YAML openshift templates + :param labels: list of labels + :param volume_list: volume list + :param volume_names: volume names + :param env_list: env list + :param ports_list: port list + :return: template with all data + """ + templ['metadata']['name'] = self.docker_image + templ['metadata']['labels'] = labels + for obj in templ['objects']: + obj['spec']['dockerImageRepository'] = self.docker_image + obj['metadata']['name'] = self.docker_image + if 'template' in obj['spec']: + obj['spec']['template']['metadata']['labels']['name'] = self.docker_image + containers = obj['spec']['template']['spec']['containers'][0] + if env_list: + containers["env"] = env_list + else: + containers.pop("env") + if ports_list: + containers["ports"] = ports_list + else: + containers.pop("ports") + if volume_list: + containers['volumeMounts'] = volume_list + obj['spec']['template']['spec']['volumes'] = volume_names + else: + containers.pop('volumeMounts') + obj['spec']['template']['spec'].pop('volumes') + containers['name'] = self.docker_image + containers['image'] = self.docker_image + + if 'triggers' in obj['spec']: + for trig in obj['spec']['triggers']: + trig['imageChangeParams']['containerNames'] = [self.docker_image] + trig['imageChangeParams']['from']['name'] = self.docker_image + ":latest" + + return templ + + def run(self): + """ + Main function + :return: + """ + self._get_openshift_template() + if not self._exist_docker_file() or not self._exist_openshift_template(): + return 1 + self._get_docker_tags() + templ = self._load_oc_template() + try: + tmpl = self.generate_oc_template(templ, *self.get_docker_directives(templ)) + except KeyError: + return 1 + self.write_oc_template(tmpl) + + diff --git a/modtools b/modtools deleted file mode 100755 index 5319e2f..0000000 --- a/modtools +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python3 - -import sys -from modularity.cli import ModtoolsCLIHelper - - -if __name__ == "__main__": - cli = ModtoolsCLIHelper() - sys.exit(cli.run()) diff --git a/modularity/__init__.py b/modularity/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/modularity/__init__.py +++ /dev/null diff --git a/modularity/cli.py b/modularity/cli.py deleted file mode 100644 index db30a7f..0000000 --- a/modularity/cli.py +++ /dev/null @@ -1,115 +0,0 @@ -import sys -import argparse -import logging - -from modularity.module_generator import ModuleGenerator -from modularity.oc_template import OpenShiftTemplateGenerator -from modularity.mod2dockerfile import ModulemdDockerfileGenerator - - -class ModtoolsCLI(object): - """ Class for processing data from commandline """ - - @staticmethod - def build_parser(): - parser = argparse.ArgumentParser(description="Generates module related files") - base_parser = argparse.ArgumentParser(add_help=False) - base_parser.add_argument( - "--verbose", - "-v", - action="store_true", - help="verbose operation" - ) - - subparsers = parser.add_subparsers(dest="cmd_name") - - parser_rpm2module = subparsers.add_parser( - 'rpm2module', parents=[base_parser], - help="Generates modulemd file", - description="Gets package info and dependencies and creates modulemd file." - ) - parser_rpm2module.add_argument( - "pkgs", - metavar='PKGS', - nargs='+', - help="Specify list of packages for module.", - ) - - parser_docker2openshift = subparsers.add_parser( - "docker2openshift", parents=[base_parser], - help="Generates openshift template from dockerfile", - description="Creates an OpenShift template YAML file.", - ) - - parser_docker2openshift.add_argument( - "image", - metavar='IMAGE', - help="docker image name (like NAME or docker.io/USER/NAME)", - ) - parser_docker2openshift.add_argument( - "--dockerfile", - help="Specify Dockerfile name. Default is Dockerfile." - ) - - parser_module2dockerfile = subparsers.add_parser( - 'module2dockerfile', parents=[base_parser], - help="Generates dockerfile from modulemd file", - description="Creates Dockerfile with suggestions and pre-filled values" - ) - - parser_module2dockerfile.add_argument( - "modulemd_file", - metavar='MODULEMD_FILE', - help='Specify path to modulemd file' - ) - - parser_module2dockerfile.add_argument( - "--output", - nargs='?', - help='Specify custom output file.' - ) - - parser_module2dockerfile.add_argument( - "--template", - help='Specify custom Dockerfile template', - required=True - ) - - return parser - - def __init__(self, args=None): - self.parser = ModtoolsCLI.build_parser() - self.args = self.parser.parse_args(args) - - def __getattr__(self, name): - try: - return getattr(self.args, name) - except AttributeError: - return object.__getattribute__(self, name) - - -class ModtoolsCLIHelper(object): - - @staticmethod - def run(): - try: - cli = ModtoolsCLI(sys.argv[1:]) - if cli.args.verbose: - logging.basicConfig(level=logging.INFO) - if cli.args.cmd_name == 'rpm2module': - mg = ModuleGenerator(cli.args.pkgs) - mg.run() - - if cli.args.cmd_name == 'docker2openshift': - otg = OpenShiftTemplateGenerator(cli.args) - otg.run() - - if cli.args.cmd_name == 'module2dockerfile': - mtd = ModulemdDockerfileGenerator(cli.args) - mtd.run() - - except KeyboardInterrupt: - print('\nInterrupted by user') - except Exception as e: - print(e) - sys.exit(1) diff --git a/modularity/mod2dockerfile.py b/modularity/mod2dockerfile.py deleted file mode 100644 index dafee67..0000000 --- a/modularity/mod2dockerfile.py +++ /dev/null @@ -1,49 +0,0 @@ -import modulemd -import os -import tempfile -import shutil -from string import Template - - -class PercentTemplate(Template): - delimiter = '%%' - - -class ModulemdDockerfileGenerator(object): - - def __init__(self, args=None): - self.dockerfile_path = args.output - self.modulemd_path = args.modulemd_file - self.template_path=args.template - - def load_modulemd(self): - self.mmd = modulemd.ModuleMetadata() - self.mmd.load(self.modulemd_path) - - self.values = {'NAME': self.mmd.name,'VERSION': self.mmd.version, - 'DESCRIPTION': self.mmd.description, 'SUMMARY': self.mmd.summary, - 'API_PACKAGES': ' '.join(self.mmd.api.rpms)} - - def generate_dockerfile(self): - tmplFile = open(self.template_path) - tmpl = PercentTemplate(tmplFile.read()) - self.result = tmpl.safe_substitute(self.values) - - def save_result(self): - if self.dockerfile_path is not None: - output_file = self.dockerfile_path - else: - tmp_dir = tempfile.mkdtemp() - if os.path.isdir(tmp_dir): - shutil.rmtree(tmp_dir) - os.makedirs(tmp_dir) - output_file = os.path.join(tmp_dir, os.path.basename('Dockerfile')) - - with open(output_file, 'w') as f: - f.write(self.result) - print("Modulemd template is generated here: %s" % (output_file)) - - def run(self): - self.load_modulemd() - self.generate_dockerfile() - self.save_result() diff --git a/modularity/module_deps_differ.py b/modularity/module_deps_differ.py deleted file mode 100755 index 4403b81..0000000 --- a/modularity/module_deps_differ.py +++ /dev/null @@ -1,237 +0,0 @@ -import os -import logging - -import dnf - -from distutils.version import LooseVersion -from pdc_client import PDCClient - - -FALLBACK_STREAM = 'master' -STREAM = 'f26' -REPO_F26 = "http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/x86_64/os/" -REPO_F26_SOURCE = "http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/source/tree/" -ARCH = 'x86_64' - -# TODO: add ability to solve more packages in one run -# TODO: solve architecture! - as option? - - -p = None - - -def get_pkgs_source_rpm_name(pkg): - """ - pkg: dnf.Package - - returns name of source RPM used to build the provided rpm - """ - if LooseVersion(dnf.__version__) < LooseVersion("2.0.0"): - source_rpm = pkg.sourcerpm - source_rpm_name = source_rpm.rsplit('-', 2)[0] - else: - source_rpm_name = pkg.source_name - return source_rpm_name - - -def get_pdc_client(): - """ cache PDCClient instance """ - global p - if p is None: - p = PDCClient('https://pdc.fedoraproject.org/rest_api/v1/', develop=True, ssl_verify=True) - return p - - -class ModuleDepsDiffer(object): - - def __init__(self, pkgs): - self.p = get_pdc_client() - self.repos = dict() - self.repo_provides = dict() - # TODO: get this from args - self.packages = list(pkgs) - self.build_req = set() - self.runtime_req = set() - self.build_caps_classified = dict() - self.build_caps_classified['result'] = set() - self.runtime_caps_classified = dict() - self.runtime_caps_classified['result'] = set() - self.repo_bases = {} - self.module_id_to_name = {} - - def get_module_name(self, koji_tag_name): - return self.module_id_to_name.get(koji_tag_name, koji_tag_name) - - def obtain_module_names(self): - logging.info('Scanning built modules, this might take a while.') - j = self.p['unreleasedvariants']( - variant_type="module", - active=True, - page_size=-1 - # variant_version=STREAM, - ) - logging.info('Scanning built modules - DONE.') - for module in j: - self.module_id_to_name[module["koji_tag"]] = "{}:{}".format(module["variant_id"], module["variant_version"]) - - @staticmethod - def get_base_from_repo(reponame, repourl): - logging.info('Loading repo: %s', reponame) - base = dnf.Base() - if LooseVersion(dnf.__version__) < LooseVersion("2.0.0"): - repo = dnf.repo.Repo(reponame, base.conf.cachedir) - else: - repo = dnf.repo.Repo(reponame, base.conf) - repo.baseurl = repourl - repo.load() - repo.enable() - base.repos.add(repo) - base.fill_sack(load_available_repos=True, load_system_repo=False) - return base - - def _init_repo_bases(self): - """ - version 1 is no longer usable since koji no longer creates repos for - modules this version iterates over repos present in - ~/modulebuild/cache/koji_tags, this means that you should do `mbs-build - local` to populate the directory - - Once we have real compose of boltron, we might utilize it here - - initialize repo objects for dnf to kick off queries - """ - logging.warn("Getting module information from mbs cache." + - " Please run mbs-build local with dependency modules" + - " you are interested in before running this script.") - cache_path = os.path.expanduser("~/modulebuild/cache/koji_tags") - for module_name in os.listdir(cache_path): - # for now ignore all what is not in f26 stream, remove when situation changes - if not self.module_id_to_name[module_name].endswith(':f26'): - continue - if module_name != "f26-modularity": # bootstrap contains almost everything, so let's ignore it - self.repos[module_name] = "file://{}".format(os.path.join(cache_path, module_name)) - for reponame, repourl in self.repos.items(): - base = ModuleDepsDiffer.get_base_from_repo(reponame, repourl) - self.repo_bases[reponame] = base - - def get_package_requires(self): - # runtime - logging.info('Getting package requirements') - pkgs = [] - base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26) - for pkg in self.packages: - filter_result = base.sack.query().filter(name=pkg, arch=['noarch', ARCH], latest=True) - ModuleDepsDiffer.package_unique(filter_result) - pkgs.append(filter_result[0]) - for pkg in pkgs: - requires = getattr(pkg, 'requires') - for q in requires: - self.runtime_req.add(str(q)) - - # build - source_pkgs = [] - base = ModuleDepsDiffer.get_base_from_repo('f26-source', REPO_F26_SOURCE) - for pkg in pkgs: - source_rpm_name = get_pkgs_source_rpm_name(pkg) - filter_result = base.sack.query().filter(name=source_rpm_name) - ModuleDepsDiffer.package_unique(filter_result) - source_pkgs.append(filter_result[0]) - for pkg in source_pkgs: - requires = getattr(pkg, 'requires') - for q in requires: - self.build_req.add(str(q)) - - # Igor suggests to do complete dependency solving here: - # base.install('package'), installroot=, make sure the right repos are enabled - # base.resolve() - # and then analyze base.transaction.install_set - # weak deps will get resolved correctly with this approach - def classify_caps(self, caps_classified, requirements): - for cap in requirements: - found = False - for reponame, base in self.repo_bases.items(): - q = base.sack.query() - pkg = q.filter(provides=[cap], arch=['noarch', ARCH], latest=True) - if pkg: - found = True - caps_classified.setdefault(reponame, set()) - caps_classified[reponame].add(cap) - # don't break here in case one cap is present in multiple repos - if not found: - caps_classified['result'].add(cap) - - @staticmethod - def package_unique(result): - if len(result) > 1: - raise ValueError('Name of package is not unique: ' + str(result.result)) - if len(result) == 0: - raise ValueError('No package found in repo') - - @staticmethod - def whatprovides(caps): - result = set() - base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26) - q = base.sack.query() - for cap in caps: - logging.info('Getting source package for %s:', cap) - pkg = q.filter(provides__glob=[cap], arch=['noarch', ARCH], latest=True) - if not pkg: - pkg = q.filter(file__glob=cap) - if len(pkg) == 0: - raise ValueError('No package provides capability \'' + cap + '\'') - source_rpm_name = get_pkgs_source_rpm_name(pkg.result[0]) - result.add(source_rpm_name) - return result - - def _dump_caps_result(self, items): - for repo, caps in items: - if not caps: - continue - if repo == 'result': - print("Capabilities which aren't provided by any module:") - else: - print("Components which are part of " + - self.get_module_name(repo) + - " module:") - for cap in sorted(caps): - print(cap) - print('') - - def dump_caps_result(self): - print('BUILD requirements:') - print('-------------------') - self._dump_caps_result(self.build_caps_classified.items()) - print('\n') - print('RUNTIME requirements:') - print('---------------------') - self._dump_caps_result(self.runtime_caps_classified.items()) - - def _dump_pkgs_result(self, items): - for repo, caps in items: - if not caps: - continue - if repo == 'result': - print("Components which aren't provided by any module:") - else: - print("Components which are part of " + - self.get_module_name(repo) + - " module:") - for cap in sorted(ModuleDepsDiffer.whatprovides(caps)): - print(cap) - print('') - - def dump_pkgs_result(self): - print('BUILD dependencies:') - print('-------------------') - self._dump_pkgs_result(self.build_caps_classified.items()) - print('\n') - print('RUNTIME dependencies:') - print('---------------------') - self._dump_pkgs_result(self.runtime_caps_classified.items()) - - def run(self): - self.obtain_module_names() - self._init_repo_bases() - self.get_package_requires() - self.classify_caps(self.build_caps_classified, self.build_req) - self.classify_caps(self.runtime_caps_classified, self.runtime_req) diff --git a/modularity/module_generator.py b/modularity/module_generator.py deleted file mode 100644 index c8932ce..0000000 --- a/modularity/module_generator.py +++ /dev/null @@ -1,120 +0,0 @@ -from __future__ import absolute_import - -import modulemd -import dnf -from .module_deps_differ import ModuleDepsDiffer -import logging - -class ModuleGenerator(object): - - def __init__(self, pkgs): - self.pkgs = pkgs - self.pkg = None - self.mmd = modulemd.ModuleMetadata() - self.build_deps = set() - self.run_deps = set() - self.differ = ModuleDepsDiffer(pkgs) - - def _save_module_md(self): - """ - Function saves modulemd file to the current directory - based on argument name - :return: - """ - - if len(self.pkgs) == 1: - file_name = self.pkgs[0] + '.yaml' - else: - file_name = "modulemd-output.yaml" - self.mmd.dump(file_name) - print('Modulemd file is generated here ./%s' % file_name) - return True - - def _update_module_md(self): - """ - Function updates modulemd file with dependencies - are information taken from SPEC file. - :return: - """ - self.mmd.add_module_license("MIT") - - if len(self.pkgs) == 1: - self.mmd.summary = str(self.pkg.summary) - self.mmd.description = str(self.pkg.description) - - # Default license for the module metadata, same as default Fedora - # content license. - - self.mmd.add_content_license(str(self.pkg.license)) - - for pkg in self.pkgs: - self.mmd.api.add_rpm(pkg) - self.mmd.components.add_rpm(ModuleDepsDiffer.whatprovides([pkg]).pop(), "Package in api", buildorder=self._get_build_order(pkg)) - - for pkg in (self.build_deps - self.mmd.api.rpms - self.run_deps): - self.mmd.filter.add_rpm(pkg) - - for pkg in self.build_deps.intersection(self.run_deps): - self.mmd.components.add_rpm(pkg, "Build and runtime dependency.", buildorder=self._get_build_order(pkg)) - - for pkg in (self.build_deps - self.run_deps): - self.mmd.components.add_rpm(pkg, "Build dependency.", buildorder=self._get_build_order(pkg)) - - for pkg in (self.run_deps - self.build_deps): - self.mmd.components.add_rpm(pkg, "Runtime dependency.", buildorder=self._get_build_order(pkg)) - - for mod, caps in self.differ.build_caps_classified.items(): - if mod == 'result': - continue - if caps: - name, stream = self.differ.get_module_name(mod).split(':') - self.mmd.add_buildrequires(name, stream) - - for mod, caps in self.differ.runtime_caps_classified.items(): - if mod == 'result': - continue - if caps: - name, stream = self.differ.get_module_name(mod).split(':') - self.mmd.add_requires(name, stream) - - def _get_build_order(self, pkg): - if pkg in self.mmd.api.rpms: - return 10 - else: - return 0 - - def _get_pkg_info(self): - """ - Function loads package from dnf - :return: - """ - logging.info("Getting package info from DNF") - b = dnf.Base() - b.read_all_repos() - b.fill_sack() - - q = b.sack.query().filter(name=self.pkgs, reponame='fedora', latest=True) - - if len(q) > 1: - raise ValueError('Name of package is not unique') - if len(q) == 0: - raise ValueError('No package found in repo') - self.pkg = q[0] - - def _get_dependencies(self): - """ - Function gets build and runtime dependencies of package - :return: - """ - logging.info('Dependency resolution started') - self.differ.run() - self.build_deps = ModuleDepsDiffer.whatprovides(self.differ.build_caps_classified['result']) - self.run_deps = ModuleDepsDiffer.whatprovides(self.differ.runtime_caps_classified['result']) - logging.info('Dependency resolution finished succesfully.') - - def run(self): - if len(self.pkgs) == 1: - self._get_pkg_info() - self._get_dependencies() - self._update_module_md() - self._save_module_md() diff --git a/modularity/oc_template.py b/modularity/oc_template.py deleted file mode 100644 index 10b803e..0000000 --- a/modularity/oc_template.py +++ /dev/null @@ -1,335 +0,0 @@ -from __future__ import absolute_import, print_function - -import os -import ast -import yaml -import tempfile -import shutil -import re -import shlex - -from dockerfile_parse import DockerfileParser - -# Dockerfile path -DOCKERFILE = "Dockerfile" - -EXPOSE = "EXPOSE" -VOLUME = "VOLUME" -LABEL = "LABEL" -ENV = "ENV" -PORTS = "PORTS" - -# OpenShift template -OPENSHIFT_TEMPLATE = "openshift-template.yml" - - -def get_string(value): - return ast.literal_eval(value) - - -class OpenShiftTemplateGenerator(object): - """ - Class generates an OpenShift template - It requires openshift-template.yml file. - """ - - dockerfile = None - oc_template = None - docker_dict = {} - - def __init__(self, args=None, dir_name=None): - if dir_name is None: - self.dir = os.getcwd() - else: - self.dir = dir_name - self.docker_image = args.image - if args.dockerfile is None: - self.dockerfile = 'Dockerfile' - else: - self.dockerfile = os.path.join(self.dir, args.dockerfile) - self.docker_dict = {} - - def _exist_docker_file(self): - """ - Function checks if docker file exists - :return: True if exists - """ - if not os.path.exists(self.dockerfile): - print("Dockerfile has to exists in the %s directory." % self.dir) - return False - return True - - def _exist_openshift_template(self): - """ - Function checks if openshift template exists - :return: True if exists - """ - if self.oc_template is None: - print("%s has to exists in the %s directory." % (OPENSHIFT_TEMPLATE, self.dir)) - return False - return True - - def _get_openshift_template(self): - """ - Function sets openshift template. - """ - for f in os.listdir(self.dir): - if os.path.isdir(os.path.join(self.dir, f)): - continue - file_name = os.path.join(self.dir, f) - if f == OPENSHIFT_TEMPLATE: - self.oc_template = file_name - - def _get_expose(self, value): - """Function returns exposes as field""" - return value.split() - - def _get_env(self, value): - """Function gets env as field""" - return shlex.split(value) - - def _get_volume(self, value): - """Function evaluates a value and returns as string.""" - return get_string(value) - - def _get_label(self, value): - """ - Function returns label from Docker file - except INSTALL, UNINSTALL and RUN label used by atomic. - :param value: row from Dockerfile - :return: label_dict - """ - untracked_values = ['INSTALL', 'UNINSTALL', 'RUN'] - if [f for f in untracked_values if value.startswith(f)]: - return None - labels = re.sub('\s\s+', ';', value).split(';') - labels = [l.replace('"', '') for l in labels] - label_dict = {} - for l in labels: - if len(l.split('=')) == 2: - label_dict[l.split('=')[0]] = l.split('=')[1] - elif re.match('maintainer', l, re.I): - label_dict['maintainer'] = l.split(' ',1)[1] - else: - raise ValueError("Unrecogised label: ", l) - return label_dict - - def _get_docker_tags(self): - """ - Function analyses dockerfile and extracts - ENV, VOLUME, EXPOSE and LABEL directives. - """ - if not self._exist_docker_file(): - return - tmp_dir = tempfile.mkdtemp() - if os.path.isdir(tmp_dir): - shutil.rmtree(tmp_dir) - os.makedirs(tmp_dir) - shutil.copyfile(self.dockerfile, os.path.join(tmp_dir, "Dockerfile")) - dfp = DockerfileParser(path=tmp_dir) - inst = "instruction" - allowed_tags = [ENV, EXPOSE, VOLUME, LABEL] - functions = {ENV: self._get_env, - EXPOSE: self._get_expose, - VOLUME: self._get_volume, - LABEL: self._get_label} - - for struct in dfp.structure: - key = struct[inst] - val = struct["value"] - if key in allowed_tags: - if key == LABEL: - if key not in self.docker_dict: - self.docker_dict[key] = {} - value = functions[key](val) - if value is not None: - self.docker_dict[key].update(value) - else: - if key not in self.docker_dict: - self.docker_dict[key] = [] - ret_val = functions[key](val) - for v in ret_val: - if v not in self.docker_dict[key]: - self.docker_dict[key].append(v) - - shutil.rmtree(tmp_dir) - - def _load_oc_template(self): - """ - Function loads openshift template - :return: YAML dictionary - """ - if not self._exist_openshift_template(): - return None - with open(self.oc_template, 'r') as f: - try: - templ = yaml.load(f) - except yaml.YAMLError as exc: - print(exc) - raise - return templ - - def _get_labels(self, templ): - labels = None - try: - labels = templ['metadata']['labels'] - except KeyError: - labels = {} - raise_exception = False - try: - labels['description'] = self.docker_dict[LABEL]['description'] - except KeyError: - print("Label Description is missing in Dockerfile. It is mandatory.") - raise_exception = True - try: - labels['tags'] = self.docker_dict[LABEL]['io.openshift.tags'] - except KeyError: - print('Label tags is missing in Dockerfile. It is mandatory.') - raise_exception = True - if raise_exception: - raise KeyError - labels['template'] = self.docker_image - return labels - - def _get_docker_labels(self): - """ - Function returns docker labels - :return: label dictionary - """ - if LABEL in self.docker_dict and self.docker_dict[LABEL]: - return self.docker_dict[LABEL] - return None - - def _get_docker_volumes(self): - """ - Function returns docker volumes and labels - :return: volume list, volume names - """ - volume_list = [] - volume_names = [] - if VOLUME in self.docker_dict and self.docker_dict[VOLUME]: - for p in self.docker_dict[VOLUME]: - volume_list.append({'mountPath': p, - 'name': 'name' + p.replace('/', '-')}) - volume_names.append({'name': 'name' + p.replace('/', '-'), - 'emptyDir': {} - }) - return volume_list, volume_names - - def _get_docker_env(self): - """ - Function return docker ENV directives - :return: list of ENV variables - """ - env_list = [] - if ENV in self.docker_dict and self.docker_dict[ENV]: - for e in self.docker_dict[ENV]: - key, val = e.split('=') - env_list.append({'name': key, - 'value': val}) - return env_list - - def _get_docker_expose(self): - """ - Function return docker EXPOSE directives - :return: list of PORTS - """ - ports_list = [] - if EXPOSE in self.docker_dict and self.docker_dict[EXPOSE]: - for p in self.docker_dict[EXPOSE]: - ports_list.append({'containerPort': int(p)}) - return ports_list - - def write_oc_template(self, templ): - """ - Function writes a YAML dictionary into template - :param templ: YAML template with all data - :return: - """ - tmp_dir = tempfile.mkdtemp() - if os.path.isdir(tmp_dir): - shutil.rmtree(tmp_dir) - os.makedirs(tmp_dir) - tmp_file = os.path.join(tmp_dir, os.path.basename(self.oc_template)) - with open(tmp_file, 'w') as f: - try: - yaml.safe_dump(templ, f, default_flow_style=False) - print("OpenShift template is generated here: %s" % (tmp_file)) - except yaml.YAMLError as exc: - print(exc) - raise - - def get_docker_directives(self, templ): - """ - Function collects all directives - :param templ: - :return: label_list, volume_list, volume_names, env_list, ports_list - """ - labels = volume_list = volume_names = env_list = ports_list = None - if self.docker_dict: - labels = self._get_labels(templ) - volume_list, volume_names = self._get_docker_volumes() - env_list = self._get_docker_env() - ports_list = self._get_docker_expose() - return labels, volume_list, volume_names, env_list, ports_list - - def generate_oc_template(self, templ, labels, volume_list, volume_names, env_list, ports_list): - """ - Function fulfills template with data taken from Dockerfile. - :param templ: YAML openshift templates - :param labels: list of labels - :param volume_list: volume list - :param volume_names: volume names - :param env_list: env list - :param ports_list: port list - :return: template with all data - """ - templ['metadata']['name'] = self.docker_image - templ['metadata']['labels'] = labels - for obj in templ['objects']: - obj['spec']['dockerImageRepository'] = self.docker_image - obj['metadata']['name'] = self.docker_image - if 'template' in obj['spec']: - obj['spec']['template']['metadata']['labels']['name'] = self.docker_image - containers = obj['spec']['template']['spec']['containers'][0] - if env_list: - containers["env"] = env_list - else: - containers.pop("env") - if ports_list: - containers["ports"] = ports_list - else: - containers.pop("ports") - if volume_list: - containers['volumeMounts'] = volume_list - obj['spec']['template']['spec']['volumes'] = volume_names - else: - containers.pop('volumeMounts') - obj['spec']['template']['spec'].pop('volumes') - containers['name'] = self.docker_image - containers['image'] = self.docker_image - - if 'triggers' in obj['spec']: - for trig in obj['spec']['triggers']: - trig['imageChangeParams']['containerNames'] = [self.docker_image] - trig['imageChangeParams']['from']['name'] = self.docker_image + ":latest" - - return templ - - def run(self): - """ - Main function - :return: - """ - self._get_openshift_template() - if not self._exist_docker_file() or not self._exist_openshift_template(): - return 1 - self._get_docker_tags() - templ = self._load_oc_template() - try: - tmpl = self.generate_oc_template(templ, *self.get_docker_directives(templ)) - except KeyError: - return 1 - self.write_oc_template(tmpl) - - diff --git a/setup.py b/setup.py index d232ee2..aa233c8 100644 --- a/setup.py +++ b/setup.py @@ -1,26 +1,26 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- -try: - from setuptools import setup, find_packages -except ImportError: - from distutils.core import setup +from setuptools import setup, find_packages setup( - name='modtools', + name='fedmod', version='0.0.1', author='Dominika Hodovska', author_email='dhodovsk@redhat.com', - description='Utilities for creating and managing modules', - long_description='Modtools now provides tools generating openshift templates from module Dockerfiles' - 'and creating modulemd files from package names (intended api of module).', + maintainer='Nick Coghlan', + maintainer_email='ncoghlan@redhat.com', + description='Utilities for generating & maintaining modulemd files', + long_description=( + "fedmod provides tools for converting existing RPMs (most notably " + "metapackages) into module definitions in Fedora's modulemd format." + ), license='MIT', keywords='modularization modularity module modulemd openshift template docker', url='https://pagure.io/modularity/modularity-tools', - scripts=['modtools'], entry_point={ 'console_scripts': [ - 'modtools=modularity.cli.ModtoolsCliHelper.run' + 'fedmod=modularity.cli.ModtoolsCliHelper.run' ] }, install_requires=[ diff --git a/tests/Dockerfile-Cockpit b/tests/Dockerfile-Cockpit new file mode 100644 index 0000000..8f729de --- /dev/null +++ b/tests/Dockerfile-Cockpit @@ -0,0 +1,31 @@ +FROM registry.fedoraproject.org/fedora:26 +MAINTAINER "Stef Walter" + +ENV VERSION=135 RELEASE=1 +LABEL BZComponent="cockpit" \ + Name="$FGC/cockpit" \ + Version="$VERSION" \ + Release="$RELEASE.$DISTTAG" \ + Architecture="x86_64" + + +RUN dnf install -y cockpit-ws cockpit-dashboard + +RUN mkdir -p /container && ln -s /host/proc/1 /container/target-namespace +ADD atomic-install /container/atomic-install +ADD atomic-uninstall /container/atomic-uninstall +ADD atomic-run /container/atomic-run +RUN chmod -v +x /container/atomic-install +RUN chmod -v +x /container/atomic-uninstall +RUN chmod -v +x /container/atomic-run + +# Make the container think it's the host OS version +RUN rm -f /etc/os-release /usr/lib/os-release && ln -sv /host/etc/os-release /etc/os-release && ln -sv /host/usr/lib/os-release /usr/lib/os-release + +LABEL INSTALL /usr/bin/docker run --rm --privileged -v /:/host IMAGE /container/atomic-install +LABEL UNINSTALL /usr/bin/docker run --rm --privileged -v /:/host IMAGE /container/atomic-uninstall +LABEL RUN /usr/bin/docker run -d --privileged --pid=host -v /:/host IMAGE /container/atomic-run --local-ssh + +# Look ma, no EXPOSE + +CMD ["/container/atomic-run"] diff --git a/tests/test_module_generator.py b/tests/test_module_generator.py index 49662bf..1433555 100644 --- a/tests/test_module_generator.py +++ b/tests/test_module_generator.py @@ -1,7 +1,7 @@ import pytest import os.path -from modularity.cli import ModtoolsCLI -from modularity.module_generator import ModuleGenerator +from fedmod.cli import ModtoolsCLI +from fedmod.module_generator import ModuleGenerator class TestSinglePackageInput(object): diff --git a/tests/test_oc_template.py b/tests/test_oc_template.py index 6391d6b..d14a30c 100644 --- a/tests/test_oc_template.py +++ b/tests/test_oc_template.py @@ -6,11 +6,11 @@ import tempfile import shutil import os import six -import urllib +import six.moves.urllib.request as urllib -from modularity.cli import ModtoolsCLI -from modularity.oc_template import OpenShiftTemplateGenerator -from modularity.oc_template import VOLUME, ENV, EXPOSE, LABEL +from fedmod.cli import ModtoolsCLI +from fedmod.oc_template import OpenShiftTemplateGenerator +from fedmod.oc_template import VOLUME, ENV, EXPOSE, LABEL def init_oc_template_generator(dockerfile, image_name, working_dir):