From afa7caef831e15928d19975eb1c6e40732fb5caa Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Nov 26 2016 00:02:21 +0000 Subject: add a client-side filtering option (#16) This adds a new feature. The user can specify a FILTEREXP in the config. If specified, this is expected to be a regular expression that quick-fedora-mirror and quick-fedora-hardlink will use to filter the file lists they work with. The regex is passed to `sed -i -r -e \,(expression),d` by q-f-m and `re.compile(expression)` by q-f-h (after stripping any quotes, in the latter case). (q-f-h.zsh does the same as q-f-m). This allows you to maintain a partial mirror using q-f-m. Items filtered out by the regex are treated as if they simply didn't exist on the remote end. If any are present locally they'll be treated as 'stale' files and removed unless skipdelete is set. They will never be synchronized. --- diff --git a/quick-fedora-hardlink b/quick-fedora-hardlink index 17bfc64..d078a05 100755 --- a/quick-fedora-hardlink +++ b/quick-fedora-hardlink @@ -89,8 +89,8 @@ def uprint(string): def config_read(opts): """Read the configuration file. - We really only want DESTD and FILELIST - The value is stored in the existing options array. + We really only want DESTD, FILELIST and FILTEREXP + The values are stored in the existing options array. """ with open(opts.config) as cf: for line in cf: @@ -102,6 +102,12 @@ def config_read(opts): if m: opts.filelist = m.group(1) + m = re.search('^\s*FILTEREXP=(\S+)', line) + if m: + opts.filterexp = re.compile(m.group(1).strip('\'"')) + else: + opts.filterexp = None + def config_check(opts): if not opts.destd: @@ -124,7 +130,7 @@ def skip_to_files(f): return False -def process_one_filelist(filelist, module, candidates, check_ctime=True): +def process_one_filelist(filelist, module, candidates, check_ctime=True, filterexp=None): with open(filelist) as f: skip_to_files(f) @@ -135,6 +141,8 @@ def process_one_filelist(filelist, module, candidates, check_ctime=True): fields = line.split('\t') if fields[1] != 'f': continue + if filterexp and filterexp.search(line): + continue ctime = fields[0] size = fields[2] @@ -160,7 +168,7 @@ def trim_candidates(candidates): del candidates[k] -def generate_candidates(filelist='fullfiletimelist-$mdir', check_ctime=True): +def generate_candidates(filelist='fullfiletimelist-$mdir', check_ctime=True, filterexp=None): """Process filelists in the current directory. Looks for all filelists in the current directory and processes them all. @@ -176,7 +184,7 @@ def generate_candidates(filelist='fullfiletimelist-$mdir', check_ctime=True): continue print(' {}'.format(fl)) - process_one_filelist(fl, module, candidates, check_ctime=check_ctime) + process_one_filelist(fl, module, candidates, check_ctime=check_ctime, filterexp=filterexp) return candidates @@ -341,7 +349,7 @@ def main(): os.chdir(opts.destd) timestart = time.time() print('Processing file lists in {}:'.format(opts.destd)) - c = generate_candidates(filelist=opts.filelist, check_ctime=opts.check_ctime) + c = generate_candidates(filelist=opts.filelist, check_ctime=opts.check_ctime, filterexp=opts.filterexp) trim_candidates(c) print() print('Found {} tuples of potentially hardlinkable files in {:.1f} minutes.' diff --git a/quick-fedora-hardlink.zsh b/quick-fedora-hardlink.zsh index 2eac216..9f698ba 100755 --- a/quick-fedora-hardlink.zsh +++ b/quick-fedora-hardlink.zsh @@ -135,6 +135,11 @@ for mdir in *; do } ' $flname > $tmpname + # filter the list, if necessary + if [[ -n $FILTEREXP ]]; then + sed -i -r -e "\,$FILTEREXP,d" $tmpname + fi + filelists+=($tmpname) linecount=$(wc -l < $tmpname) totallines=$((totallines + linecount)) diff --git a/quick-fedora-mirror b/quick-fedora-mirror index 3076c6f..ed67ff3 100755 --- a/quick-fedora-mirror +++ b/quick-fedora-mirror @@ -76,6 +76,13 @@ finish () { exit 0 } +# Client-side file list filtering. +filter () { + if [[ -n $FILTEREXP ]]; then + sed -i -r -e "\,$FILTEREXP,d" $1 + fi +} + do_rsync () { # Options: source, destination. # The name of an array containing additional options. @@ -462,6 +469,11 @@ for module in $MODULES; do {if (s && \$2 == \"f\" || \$2 == \"l\") print \"$moduledir/\" \$4 \"\t\" \$3} " $fl > allfilesizes-$module + + # Filter the list if requested. + filter allfilesizes-$module + + # Produce the file list. awk -F '\t' '{print $1}' allfilesizes-$module > allfiles-$module # All dirs in the remote repository. @@ -470,6 +482,9 @@ for module in $MODULES; do print \"$moduledir/\" \$4} " < $fl > alldirs-$module + # Filter the list if requested. + filter alldirs-$module + linecount=$(wc -l < allfiles-$module) linecount2=$(wc -l < alldirs-$module) db2f "Total on server: %7d files, %4d dirs.\n" $linecount $linecount2 @@ -480,12 +495,18 @@ for module in $MODULES; do print \"$moduledir/\" \$4} " < $fl > newfiles-$module + # Filter the list if requested. + filter newfiles-$module + # Dirs on the server which changed since the last run awk -F '\t' "/\\[Files/ {s=1;next} /^\$/ {s=0;next} { if (s && \$1 >= $LASTTIME &&(\$2 == \"d\")) print \"$moduledir/\" \$4} " < $fl > newdirs-$module + # Filter the list if requested. + filter newdirs-$module + linecount=$(wc -l < newfiles-$module) linecount2=$(wc -l < newdirs-$module) db2f "New on server: %7d files, %4d dirs.\n" $linecount $linecount2 @@ -639,7 +660,6 @@ if [[ -n $MIRRORBUFFET ]]; then echo DIRECTORY_SIZES.txt >> master-transferlist fi - # The actual transfer # =================== sort -u master-transferlist > master-transferlist.sorted diff --git a/quick-fedora-mirror.conf.dist b/quick-fedora-mirror.conf.dist index cca8c47..b198846 100644 --- a/quick-fedora-mirror.conf.dist +++ b/quick-fedora-mirror.conf.dist @@ -133,3 +133,14 @@ TIMEFILE= # about the server potentially sending extra-huge files and filling things up, # you can set TMPDIR here. # TMPDIR= + +# A regular expression used to filter the file lists. It must be quoted (or +# very carefully escaped). Entries matching this expression will not be synced +# and are expected not to be present locally. They will also be ignored by +# quick-fedora-hardlink. Cannot contain commas. Run against the file list that +# includes sizes (by quick-fedora-mirror) and the fullfiletimelist (by +# quick-fedora-hardlink), so don't use expressions that would match those +# metadata (which are digit strings and single characters). Example is a heavy +# filter which gives you an x86_64-only mirror with source packages, debuginfo +# packages, Alpha and Beta releases, and most image files excluded. +# FILTEREXP='(/i386|/armhfp|/source|/SRPMS|/debug/|\.iso|\.img|\.qcow2|\.raw\.xz|\.box|/releases/test)'