From a6bda7894ddffdf98eac9253f1f3216c5147a8c2 Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Nov 22 2016 06:43:38 +0000
Subject: change 'filterlist' to 'imagelist', use productmd formats


As an alternative to filtering out known not-image file types,
just import the list of known image file types from productmd
and filter out anything not in that when creating the list for
fedfind. Now call it 'imagelist' rather than 'filterlist'.

The advantage of this is it gives us the smallest possible list
and we don't have to worry about the filterlist file sizes
suddenly growing because a large number of some new type of
file starts showing up in the compose process (this would make
fedfind runs suddenly use a lot more bandwidth, but otherwise
wouldn't be terribly visible, which is kinda bad).

The disadvantage is it introduces a dependency on productmd
which we'd have to ensure we always account for on the systems
where create-filelist is run, and keep up to date so we don't
miss new image types.

At tibbs' request, make create-filelist work if productmd is
not available, to keep it generically useful: if productmd is
not available, the imagelist feature is disabled.

---

diff --git a/create-filelist b/create-filelist
index 8fc3367..fcbdea4 100755
--- a/create-filelist
+++ b/create-filelist
@@ -14,6 +14,12 @@ import os
 import sys
 from scandir import scandir
 
+# productmd is optional, needed only for the imagelist feature
+try:
+    from productmd.images import SUPPORTED_IMAGE_FORMATS
+except ImportError:
+    SUPPORTED_IMAGE_FORMATS = []
+
 
 def get_ftype(entry):
     """Return a simple indicator of the file type."""
@@ -58,8 +64,8 @@ def parseopts():
     null = open(os.devnull, 'w')
     p = argparse.ArgumentParser(
         description='Generate a list of files and times, suitable for consumption by quick-fedora-mirror, '
-                    'and a much smaller list with packages, Device Tree boot files, HTML files and '
-                    'directories filtered out, for consumption by fedfind.')
+                    'and (optionally) a much smaller list of only files that match one of the productmd '
+                    ' supported image types, for use by fedfind.')
     p.add_argument('-c', '--checksum', action='store_true',
                    help='Include checksums of all repomd.xml files in the file list.')
     p.add_argument('-C', '--checksum-file', action='append', dest='checksum_files',
@@ -75,8 +81,9 @@ def parseopts():
                    help='Filename of the file list with times (default: stdout).')
     p.add_argument('-f', '--filelist', type=argparse.FileType('w'), default=null,
                    help='Filename of the file list without times (default: no plain file list is generated).')
-    p.add_argument('-F', '--filterlist', type=argparse.FileType('w'), default=null,
-                   help='Filename of the filtered file list for fedfind (default: not generated).')
+    p.add_argument('-i', '--imagelist', type=argparse.FileType('w'), default=null,
+                   help='Filename of the image file list for fedfind (default: not generated). Requires '
+                   'the productmd library.')
 
     opts = p.parse_args()
 
@@ -99,6 +106,8 @@ def parseopts():
 
 def main():
     opts = parseopts()
+    if opts.imagelist.name != os.devnull and not SUPPORTED_IMAGE_FORMATS:
+        sys.exit("--imagelist requires the productmd library!")
     checksums = {}
 
     os.chdir(opts.dir)
@@ -112,9 +121,9 @@ def main():
         # opts.filelist.write(entry.path + '\n')
         print(entry.path, file=opts.filelist)
         # write to filtered list if appropriate
-        skips = ('.rpm', '.drpm', '.dtb', '.html')
-        if not any(entry.path.endswith(skip) for skip in skips) and not (entry.is_dir()):
-            print(entry.path, file=opts.filterlist)
+        imgs = ['.{0}'.format(form) for form in SUPPORTED_IMAGE_FORMATS]
+        if any(entry.path.endswith(img) for img in imgs):
+            print(entry.path, file=opts.imagelist)
         if entry.name in opts.checksum_files:
             checksums[entry.path[2:]] = True
         info = entry.stat(follow_symlinks=False)