From 2888bd324c232eb78b929ecd9f07cdb48906f796 Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Jul 15 2021 14:57:32 +0000 Subject: [PATCH 1/2] Add zipped scrambled log as test data --- diff --git a/test_data/mirrors.fedoraproject.org-access.log.processed.xz b/test_data/mirrors.fedoraproject.org-access.log.processed.xz new file mode 100644 index 0000000..44921fe Binary files /dev/null and b/test_data/mirrors.fedoraproject.org-access.log.processed.xz differ From 29cebff45b3a999018937a52370166a490ed236f Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Jul 20 2021 14:32:56 +0000 Subject: [PATCH 2/2] Add first integration test to test parsing logs This required a slight refactor of the script, so that we don't need to deal with parsing CLI options, in the test itself. The test uses a xz log-file as an input, and xz db file to compare the result to. Signed-off-by: Adam Saleh --- diff --git a/countme/parse.py b/countme/parse.py new file mode 100644 index 0000000..4556d26 --- /dev/null +++ b/countme/parse.py @@ -0,0 +1,32 @@ +from countme.progress import ReadProgress + + +def parse(args=None): + if args.header or args.sqlite: + args.writer.write_header() + + for logf in ReadProgress(args.logs, display=args.progress): + # Make an iterator object for the matching log lines + match_iter = iter(args.matcher(logf)) + + # TEMP WORKAROUND: filter out match items with missing values + if args.matchmode == "countme": + match_iter = filter(lambda i: None not in i, match_iter) + + # Duplicate data check (for sqlite output) + if args.dupcheck: + try: + item = next(match_iter) # grab first matching item + except StopIteration: + # If there is no next match, keep going + continue + if args.writer.has_item(item): # if it's already in the db... + continue # skip to next log + else: # otherwise + args.writer.write_item(item) # insert it into the db + + # Write matching items (sqlite does commit at end, or rollback on error) + args.writer.write_items(match_iter) + + if args.index: + args.writer.write_index() diff --git a/parse-access-log.py b/parse-access-log.py index 8971249..c3bec6e 100755 --- a/parse-access-log.py +++ b/parse-access-log.py @@ -27,8 +27,7 @@ import sys import argparse from countme import CountmeMatcher, MirrorMatcher, make_writer - -from countme.progress import ReadProgress +from countme.parse import parse # =========================================================================== # ====== CLI parser & main() ================================================ @@ -111,41 +110,9 @@ def parse_args(argv=None): return args -def main(): - args = parse_args() - - if args.header or args.sqlite: - args.writer.write_header() - - for logf in ReadProgress(args.logs, display=args.progress): - # Make an iterator object for the matching log lines - match_iter = iter(args.matcher(logf)) - - # TEMP WORKAROUND: filter out match items with missing values - if args.matchmode == "countme": - match_iter = filter(lambda i: None not in i, match_iter) - - # Duplicate data check (for sqlite output) - if args.dupcheck: - try: - item = next(match_iter) # grab first matching item - except StopIteration: - # If there is no next match, keep going - continue - if args.writer.has_item(item): # if it's already in the db... - continue # skip to next log - else: # otherwise - args.writer.write_item(item) # insert it into the db - - # Write matching items (sqlite does commit at end, or rollback on error) - args.writer.write_items(match_iter) - - if args.index: - args.writer.write_index() - - if __name__ == "__main__": try: - main() + args = parse_args() + parse(args) except KeyboardInterrupt: raise SystemExit(3) # sure, 3 is good, why not diff --git a/test_data/mirrors.fedoraproject.org-access.log.processed.xz b/test_data/mirrors.fedoraproject.org-access.log.processed.xz deleted file mode 100644 index 44921fe..0000000 Binary files a/test_data/mirrors.fedoraproject.org-access.log.processed.xz and /dev/null differ diff --git a/test_data/mirrors.tar.xz b/test_data/mirrors.tar.xz new file mode 100644 index 0000000..0f8d8c0 Binary files /dev/null and b/test_data/mirrors.tar.xz differ diff --git a/test_data/test_result_cmp.tar.xz b/test_data/test_result_cmp.tar.xz new file mode 100644 index 0000000..6626584 Binary files /dev/null and b/test_data/test_result_cmp.tar.xz differ diff --git a/tests/test_parse.py b/tests/test_parse.py new file mode 100644 index 0000000..9ea9d7d --- /dev/null +++ b/tests/test_parse.py @@ -0,0 +1,58 @@ +import tarfile +import os +import sqlite3 +from typing import Any, List, NamedTuple + + +from countme import CountmeMatcher, make_writer +from countme.parse import parse + + +class Args(NamedTuple): + writer: Any + matcher: Any + dupcheck: bool + index: Any + header: bool + progress: bool + matchmode: str + format: str + sqlite: str + logs: List[str] + + +def test_read_file(tmp_path): + matcher = CountmeMatcher + args = Args( + writer=make_writer("sqlite", str(tmp_path / "test_result.db"), matcher.itemtuple), + matcher=matcher, + dupcheck=True, + index=True, + header=True, + progress=False, + matchmode="countme", + format="csv", + sqlite=str(tmp_path / "test_result.db"), + logs=[str(tmp_path / "mirrors.fedoraproject.org-access.log.processed")], + ) + with tarfile.open("./test_data/mirrors.tar.xz", "r:xz") as log_tar: + with tarfile.open("./test_data/test_result_cmp.tar.xz", "r:xz") as db_tar: + os.chdir(tmp_path) + log_tar.extractall() + parse(args) + db_tar.extractall() + print(tmp_path) + db = sqlite3.connect(args.sqlite) + tmp_db = tmp_path / "test_result_cmp.db" + db.execute(f"ATTACH DATABASE '{tmp_db}' AS test_db;") + rows_missing = db.execute( + "select * from test_db.countme_raw except select * from countme_raw;" + ) + missing = rows_missing.fetchone() + rows_extra = db.execute( + "select * from countme_raw except select * from test_db.countme_raw;" + ) + extra = rows_extra.fetchone() + assert ( + missing is None and extra is None + ), f"When comparing db's\n {missing} was missing and\n {extra} was extra"