From 566696be3709a3f3cb1f37fac1d1f93c7ebe539a Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Aug 19 2021 10:06:15 +0000 Subject: [PATCH 1/3] Add tmp_path_cwd fixture and use it This wraps the tmp_path fixture of pytest, and changes into the temporary directory before each test and out of it afterwards. Additionally, it removes a superfluous print. Signed-off-by: Nils Philippsen --- diff --git a/tests/test_parse.py b/tests/test_parse.py index 9ea9d7d..2178f1c 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -1,13 +1,27 @@ import tarfile import os import sqlite3 +from pathlib import Path from typing import Any, List, NamedTuple +import pytest from countme import CountmeMatcher, make_writer from countme.parse import parse +HERE = Path(__file__).parent +TEST_DATA_DIR = HERE.parent / "test_data" + + +@pytest.fixture +def tmp_path_cwd(tmp_path): + old_wd = os.getcwd() + os.chdir(tmp_path) + yield tmp_path + os.chdir(old_wd) + + class Args(NamedTuple): writer: Any matcher: Any @@ -21,10 +35,10 @@ class Args(NamedTuple): logs: List[str] -def test_read_file(tmp_path): +def test_read_file(tmp_path_cwd): matcher = CountmeMatcher args = Args( - writer=make_writer("sqlite", str(tmp_path / "test_result.db"), matcher.itemtuple), + writer=make_writer("sqlite", str(tmp_path_cwd / "test_result.db"), matcher.itemtuple), matcher=matcher, dupcheck=True, index=True, @@ -32,18 +46,16 @@ def test_read_file(tmp_path): progress=False, matchmode="countme", format="csv", - sqlite=str(tmp_path / "test_result.db"), - logs=[str(tmp_path / "mirrors.fedoraproject.org-access.log.processed")], + sqlite=str(tmp_path_cwd / "test_result.db"), + logs=[str(tmp_path_cwd / "mirrors.fedoraproject.org-access.log.processed")], ) - with tarfile.open("./test_data/mirrors.tar.xz", "r:xz") as log_tar: - with tarfile.open("./test_data/test_result_cmp.tar.xz", "r:xz") as db_tar: - os.chdir(tmp_path) + with tarfile.open(TEST_DATA_DIR / "mirrors.tar.xz", "r:xz") as log_tar: + with tarfile.open(TEST_DATA_DIR / "test_result_cmp.tar.xz", "r:xz") as db_tar: log_tar.extractall() parse(args) db_tar.extractall() - print(tmp_path) db = sqlite3.connect(args.sqlite) - tmp_db = tmp_path / "test_result_cmp.db" + tmp_db = tmp_path_cwd / "test_result_cmp.db" db.execute(f"ATTACH DATABASE '{tmp_db}' AS test_db;") rows_missing = db.execute( "select * from test_db.countme_raw except select * from countme_raw;" From afddc2775fcb78db1286f71a35d6b612b1d50424 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Aug 19 2021 15:58:03 +0000 Subject: [PATCH 2/3] Skip test if test data tarballs aren't accessible This converts log_tar, db_tar into fixtures which extract the tarballs into the temporary path if they can be opened successfully. On failure, the fixtures yield None and the test is skipped. Signed-off-by: Nils Philippsen --- diff --git a/tests/test_parse.py b/tests/test_parse.py index 2178f1c..c47d37c 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -22,6 +22,33 @@ def tmp_path_cwd(tmp_path): os.chdir(old_wd) +def _test_tarfile_factory(tarfile_path): + """Wrap tarfile.open() context manager for fixtures + + This attempts to open the tarfile and if successful, extracts its contents + to the current working directory and yields the Tarfile object. On + failure, it yields None. + """ + try: + tarfp = tarfile.open(tarfile_path, "r:xz") + except FileNotFoundError: + yield None + else: + with tarfp: + tarfp.extractall() + yield tarfp + + +@pytest.fixture +def log_tar(tmp_path_cwd): + yield from _test_tarfile_factory(TEST_DATA_DIR / "mirrors.tar.xz") + + +@pytest.fixture +def db_tar(tmp_path_cwd): + yield from _test_tarfile_factory(TEST_DATA_DIR / "test_result_cmp.tar.xz") + + class Args(NamedTuple): writer: Any matcher: Any @@ -35,7 +62,9 @@ class Args(NamedTuple): logs: List[str] -def test_read_file(tmp_path_cwd): +def test_read_file(tmp_path_cwd, log_tar, db_tar): + if not log_tar or not db_tar: + pytest.skip("Test data not found") matcher = CountmeMatcher args = Args( writer=make_writer("sqlite", str(tmp_path_cwd / "test_result.db"), matcher.itemtuple), @@ -49,22 +78,14 @@ def test_read_file(tmp_path_cwd): sqlite=str(tmp_path_cwd / "test_result.db"), logs=[str(tmp_path_cwd / "mirrors.fedoraproject.org-access.log.processed")], ) - with tarfile.open(TEST_DATA_DIR / "mirrors.tar.xz", "r:xz") as log_tar: - with tarfile.open(TEST_DATA_DIR / "test_result_cmp.tar.xz", "r:xz") as db_tar: - log_tar.extractall() - parse(args) - db_tar.extractall() - db = sqlite3.connect(args.sqlite) - tmp_db = tmp_path_cwd / "test_result_cmp.db" - db.execute(f"ATTACH DATABASE '{tmp_db}' AS test_db;") - rows_missing = db.execute( - "select * from test_db.countme_raw except select * from countme_raw;" - ) - missing = rows_missing.fetchone() - rows_extra = db.execute( - "select * from countme_raw except select * from test_db.countme_raw;" - ) - extra = rows_extra.fetchone() - assert ( - missing is None and extra is None - ), f"When comparing db's\n {missing} was missing and\n {extra} was extra" + parse(args) + db = sqlite3.connect(args.sqlite) + tmp_db = tmp_path_cwd / "test_result_cmp.db" + db.execute(f"ATTACH DATABASE '{tmp_db}' AS test_db;") + rows_missing = db.execute("select * from test_db.countme_raw except select * from countme_raw;") + missing = rows_missing.fetchone() + rows_extra = db.execute("select * from countme_raw except select * from test_db.countme_raw;") + extra = rows_extra.fetchone() + assert ( + missing is None and extra is None + ), f"When comparing db's\n {missing} was missing and\n {extra} was extra" From 5fc70cd6e45d3bb9aa02c301a75b68cb1c7e75a4 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Aug 19 2021 15:58:05 +0000 Subject: [PATCH 3/3] Specify which files should be distributed Primarily, don't ship test_data/ which contains large tarballs. It's sufficient if these tests are run from a cloned git repository. This adds etuptools-scm as a build dependency. Signed-off-by: Nils Philippsen --- diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..9ca4a89 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +exclude .zuul.yaml +prune ci +prune test_data diff --git a/mirrors-countme.spec b/mirrors-countme.spec index b214ee0..eb0fb0f 100644 --- a/mirrors-countme.spec +++ b/mirrors-countme.spec @@ -27,7 +27,9 @@ system arch, etc.} # This section defines the python3-mirrors-countme subpackage. %package -n python3-%{srcname} Summary: %{summary} -BuildRequires: python3-devel python3-setuptools +BuildRequires: python3-devel +BuildRequires: python3-setuptools +BuildRequires: python3-setuptools_scm #Recommends: python3-%%{srcname}+fancy_progress # NOTE: in F33+ %%python_extras_subpkg can be used to automatically generate # a 'python3-mirrors-countme+fancy_progress' subpackage that would pull in the diff --git a/pyproject.toml b/pyproject.toml index 2096b4c..27e118f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,4 @@ +[build-system] + requires = ["setuptools", "setuptools-scm", "wheel"] [tool.black] line_length = 100