From a609b1cb2b43fd17e03efa62314f679b47ae6cb5 Mon Sep 17 00:00:00 2001 From: David Luevano Alvarado Date: Fri, 24 Feb 2023 03:53:13 -0600 Subject: add utils tests, small refactor --- setup.cfg | 4 +- src/pyssg/configuration.py | 22 ++-- src/pyssg/utils.py | 20 +++- tests/conftest.py | 48 ++++++++- tests/io_files/multiple.yaml | 55 ++++++++++ tests/io_files/multiple_one_doc_error.yaml | 48 +++++++++ tests/test_configuration.py | 70 +++++++------ tests/test_utils.py | 162 +++++++++++++++++++++++++++++ tests/test_yaml_parser.py | 4 +- 9 files changed, 382 insertions(+), 51 deletions(-) create mode 100644 tests/io_files/multiple.yaml create mode 100644 tests/io_files/multiple_one_doc_error.yaml create mode 100644 tests/test_utils.py diff --git a/setup.cfg b/setup.cfg index bbd073e..d61839e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,9 +57,11 @@ where = src pymdvar = py.typed [flake8] -max-line-length = 160 +max-line-length = 80 per-file-ignores = __init__.py: W292 + arg_parser.py: E501 + custom_logger.py: E501 [pbr] skip_authors = True diff --git a/src/pyssg/configuration.py b/src/pyssg/configuration.py index 3cc5430..e2dc26b 100644 --- a/src/pyssg/configuration.py +++ b/src/pyssg/configuration.py @@ -12,10 +12,11 @@ DEFAULT_CONFIG_PATH: str = '$XDG_CONFIG_HOME/pyssg/config.yaml' VERSION: str = version('pyssg') -def __check_well_formed_config(config: dict, - config_base: list[dict], +def __check_well_formed_config(config: dict[str, Any], + config_base: list[dict[str, Any]], prefix_key: str = '') -> None: for key in config_base[0].keys(): + new_config_base: list[dict[str, Any]] = [] current_key: str = f'{prefix_key}.{key}' if prefix_key != '' else key log.debug('checking "%s"', current_key) if key not in config: @@ -27,7 +28,7 @@ def __check_well_formed_config(config: dict, try: config[key].keys() except AttributeError: - log.error('config doesn\'t have any dirs configs (dirs.*)') + log.error('config doesn\'t have any dirs (dirs.*)') sys.exit(1) if '/' not in config[key]: log.debug('key: %s; config.keys: %s', key, config[key].keys()) @@ -37,7 +38,7 @@ def __check_well_formed_config(config: dict, key, ', '.join(config[key].keys())) for dkey in config[key].keys(): new_current_key: str = f'{current_key}.{dkey}' - new_config_base: list[dict] = [config_base[1], config_base[1]] + new_config_base = [config_base[1], config_base[1]] __check_well_formed_config(config[key][dkey], new_config_base, new_current_key) @@ -46,11 +47,11 @@ def __check_well_formed_config(config: dict, if not config_base[0][key]: log.debug('"%s" doesn\'t need nested elements', current_key) continue - new_config_base: list[dict] = [config_base[0][key], config_base[1]] + new_config_base = [config_base[0][key], config_base[1]] __check_well_formed_config(config[key], new_config_base, current_key) -def __expand_all_paths(config: dict) -> None: +def __expand_all_paths(config: dict[str, Any]) -> None: log.debug('expanding all path options: %s', config['path'].keys()) for option in config['path'].keys(): config['path'][option] = get_expanded_path(config['path'][option]) @@ -59,10 +60,11 @@ def __expand_all_paths(config: dict) -> None: # not necessary to type deeper than the first dict def get_parsed_config(path: str, mc_package: str = 'mandatory_config.yaml', - plt_resource: str = 'pyssg.plt') -> list[dict]: + plt_resource: str = 'pyssg.plt') -> list[dict[str, Any]]: log.debug('reading config file "%s"', path) - config_all: list[dict] = get_parsed_yaml(path) - mandatory_config: list[dict] = get_parsed_yaml(mc_package, plt_resource) + config_all: list[dict[str, Any]] = get_parsed_yaml(path) + mandatory_config: list[dict[str, Any]] = get_parsed_yaml(mc_package, + plt_resource) log.info('found %s document(s) for config "%s"', len(config_all), path) log.debug('checking that config file is well formed') for config in config_all: @@ -74,7 +76,7 @@ def get_parsed_config(path: str, # not necessary to type deeper than the first dict, # static config means config that shouldn't be changed by the user def get_static_config(sc_package: str = 'static_config.yaml', - plt_resource: str = 'pyssg.plt') -> dict[str, dict]: + plt_resource: str = 'pyssg.plt') -> dict[str, Any]: log.debug('reading and setting static config') config: dict[str, Any] = get_parsed_yaml(sc_package, plt_resource)[0] diff --git a/src/pyssg/utils.py b/src/pyssg/utils.py index 8300a5c..b1ed8c1 100644 --- a/src/pyssg/utils.py +++ b/src/pyssg/utils.py @@ -20,7 +20,8 @@ def get_file_list(path: str, dirs[:] = [d for d in dirs if d not in exclude_dirs] for file in files: if file.endswith(exts): - # [1:] is required to remove the '/' at the beginning after replacing + # [1:] is required to remove the '/' + # at the beginning after replacing file_name: str = os.path.join(root, file).replace(path, '')[1:] file_list.append(file_name) log.debug('added file "%s" without "%s" part: "%s"', @@ -44,7 +45,8 @@ def get_dir_structure(path: str, if root in dir_list: dir_list.remove(root) log.debug('removed dir "%s" as it already is in the list', root) - # not removing the 'path' part here, as comparisons with 'root' would fail + # not removing the 'path' part here, + # as comparisons with 'root' would fail joined_dir: str = os.path.join(root, d) dir_list.append(joined_dir) log.debug('added dir "%s" to the list', joined_dir) @@ -53,19 +55,29 @@ def get_dir_structure(path: str, return [d.replace(path, '')[1:] for d in dir_list] +# TODO: probably change it so it returns a bool, easier to check def create_dir(path: str, p: bool = False, silent=False) -> None: + log_msg: str = '' try: if p: os.makedirs(path) else: os.mkdir(path) + log_msg = f'created directory "{path}"' if not silent: - log.info('created directory "%s"', path) + log.info(log_msg) + log.debug(log_msg) except FileExistsError: + log_msg = f'directory "{path}" exists, ignoring' if not silent: - log.info('directory "%s" already exists, ignoring', path) + log.info(log_msg) + log.debug(log_msg) +# TODO: change this as it doesn't take directories into account, +# a file can be copied into a directory, need to get the filename +# and use it when copying +# TODO: probably change it so it returns a bool, easier to check def copy_file(src: str, dst: str) -> None: if not os.path.exists(dst): shutil.copy2(src, dst) diff --git a/tests/conftest.py b/tests/conftest.py index b44fbf6..e2d6946 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,8 @@ import os import sys import pytest -from typing import Callable +from pathlib import Path +from typing import Any, Callable from pytest import MonkeyPatch from argparse import ArgumentParser from datetime import datetime, timezone @@ -70,3 +71,48 @@ def get_fmt_time() -> Callable[..., str]: def fmt_time(fmt: str) -> str: return datetime.now(tz=timezone.utc).strftime(fmt) return fmt_time + + +@pytest.fixture +def simple_dict() -> dict[str, Any]: + return {'define': '$PYSSG_HOME/pyssg/site_example/', + 'title': 'Example site', + 'path': { + 'src': '/tmp/pyssg/pyssg/site_example/src', + 'dst': '/tmp/pyssg/pyssg/site_example/dst', + 'plt': '/tmp/pyssg/pyssg/site_example/plt', + 'db': '/tmp/pyssg/pyssg/site_example/.files'}, + 'url': { + 'main': 'https://example.com', + 'static': 'https://static.example.com', + 'default_image': 'images/default.png'}, + 'fmt': { + 'date': '%a, %b %d, %Y @ %H:%M %Z', + 'list_date': '%b %d', + 'list_sep_date': '%B %Y'}, + 'dirs': { + '/': { + 'cfg': { + 'plt': 'page.html', + 'tags': False, + 'index': False, + 'rss': False, + 'sitemap': False, + 'exclude_dirs': []}}}} + + +@pytest.fixture(scope='function') +def tmp_dir_structure(tmp_path: Path) -> Path: + root: Path = tmp_path/'dir_str' + # order matters + dirs: list[Path] = [root, + root/'first', + root/'first/f1', + root/'first/f1/f2', + root/'second', + root/'second/s1'] + for i, d in enumerate(dirs): + d.mkdir() + for ext in ['txt', 'md', 'html']: + (d/f'f{i}.{ext}').write_text('sample') + return root diff --git a/tests/io_files/multiple.yaml b/tests/io_files/multiple.yaml new file mode 100644 index 0000000..8d99c40 --- /dev/null +++ b/tests/io_files/multiple.yaml @@ -0,0 +1,55 @@ +%YAML 1.2 +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" + static: "https://static.example.com" + default_image: "images/default.png" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" + list_date: "%b %d" + list_sep_date: "%B %Y" +dirs: + /: + cfg: + plt: "page.html" + tags: False + index: False + rss: False + sitemap: False + exclude_dirs: [] +... +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" + static: "https://static.example.com" + default_image: "images/default.png" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" + list_date: "%b %d" + list_sep_date: "%B %Y" +dirs: + /: + cfg: + plt: "page.html" + tags: False + index: False + rss: False + sitemap: False + exclude_dirs: [] +... \ No newline at end of file diff --git a/tests/io_files/multiple_one_doc_error.yaml b/tests/io_files/multiple_one_doc_error.yaml new file mode 100644 index 0000000..86f6546 --- /dev/null +++ b/tests/io_files/multiple_one_doc_error.yaml @@ -0,0 +1,48 @@ +%YAML 1.2 +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" + static: "https://static.example.com" + default_image: "images/default.png" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" + list_date: "%b %d" + list_sep_date: "%B %Y" +dirs: + /: + cfg: + plt: "page.html" + tags: False + index: False + rss: False + sitemap: False + exclude_dirs: [] +... +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" + static: "https://static.example.com" + default_image: "images/default.png" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" + list_date: "%b %d" + list_sep_date: "%B %Y" +dirs: + # just removing all paths as it will cause an error +... \ No newline at end of file diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 0db8094..68f7808 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1,16 +1,16 @@ import pytest from pytest import LogCaptureFixture from typing import Any, Callable -from logging import DEBUG, INFO, WARNING, ERROR +from logging import ERROR from pyssg.configuration import get_static_config, get_parsed_config # this test is a bit sketchy, as the way the datetimes are calculated could vary # by milliseconds or even have a difference in seconds def test_static_default(rss_date_fmt: str, - sitemap_date_fmt: str, - get_fmt_time: Callable[..., str], - version: str) -> None: + sitemap_date_fmt: str, + get_fmt_time: Callable[..., str], + version: str) -> None: rss_run_date: str = get_fmt_time(rss_date_fmt) sitemap_run_date: str = get_fmt_time(sitemap_date_fmt) sc_dict: dict[str, Any] = {'fmt': {'rss_date': rss_date_fmt, @@ -22,35 +22,13 @@ def test_static_default(rss_date_fmt: str, assert static_config == sc_dict -def test_simple(test_dir: str) -> None: - yaml_dict: dict[str, Any] = {'define': '$PYSSG_HOME/pyssg/site_example/', - 'title': 'Example site', - 'path': { - 'src': '/tmp/pyssg/pyssg/site_example/src', - 'dst': '/tmp/pyssg/pyssg/site_example/dst', - 'plt': '/tmp/pyssg/pyssg/site_example/plt', - 'db': '/tmp/pyssg/pyssg/site_example/.files'}, - 'url': { - 'main': 'https://example.com', - 'static': 'https://static.example.com', - 'default_image': 'images/default.png'}, - 'fmt': { - 'date': '%a, %b %d, %Y @ %H:%M %Z', - 'list_date': '%b %d', - 'list_sep_date': '%B %Y'}, - 'dirs': { - '/': { - 'cfg': { - 'plt': 'page.html', - 'tags': False, - 'index': False, - 'rss': False, - 'sitemap': False, - 'exclude_dirs': []}}}} - yaml_path: str = f'{test_dir}/io_files/simple.yaml' +def test_simple(test_dir: str, + simple_yaml: str, + simple_dict: dict[str, Any]) -> None: + yaml_path: str = f'{test_dir}/io_files/{simple_yaml}' yaml: list[dict[str, Any]] = get_parsed_config(yaml_path) assert len(yaml) == 1 - assert yaml[0] == yaml_dict + assert yaml[0] == simple_dict def test_simple_mising_key(test_dir: str, @@ -67,10 +45,10 @@ def test_simple_mising_key(test_dir: str, def test_simple_mising_dirs(test_dir: str, - caplog: LogCaptureFixture) -> None: + caplog: LogCaptureFixture) -> None: err: tuple[str, int, str] = ('pyssg.configuration', ERROR, - 'config doesn\'t have any dirs configs (dirs.*)') + 'config doesn\'t have any dirs (dirs.*)') yaml_path: str = f'{test_dir}/io_files/simple_missing_dirs.yaml' with pytest.raises(SystemExit) as system_exit: get_parsed_config(yaml_path) @@ -90,3 +68,29 @@ def test_simple_root_dir(test_dir: str, assert system_exit.type == SystemExit assert system_exit.value.code == 1 assert caplog.record_tuples[-1] == err + + +# this really just tests that both documents in the yaml file are read, +# multiple.yaml is just simple.yaml with the same document twice, +# shouldn't be an issue as the yaml package handles this +def test_multiple(test_dir: str, simple_dict: dict[str, Any]) -> None: + yaml_path: str = f'{test_dir}/io_files/multiple.yaml' + yaml: list[dict[str, Any]] = get_parsed_config(yaml_path) + assert len(yaml) == 2 + assert yaml[0] == simple_dict + assert yaml[1] == simple_dict + + +# also, this just tests that the checks for a well formed config file are +# processed for all documents +def test_multiple_one_doc_error(test_dir: str, + caplog: LogCaptureFixture) -> None: + err: tuple[str, int, str] = ('pyssg.configuration', + ERROR, + 'config doesn\'t have any dirs (dirs.*)') + yaml_path: str = f'{test_dir}/io_files/multiple_one_doc_error.yaml' + with pytest.raises(SystemExit) as system_exit: + get_parsed_config(yaml_path) + assert system_exit.type == SystemExit + assert system_exit.value.code == 1 + assert caplog.record_tuples[-1] == err diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..b7c9754 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,162 @@ +import pytest +from pytest import LogCaptureFixture +from pathlib import Path +from logging import INFO +from pyssg.utils import (get_expanded_path, get_checksum, copy_file, create_dir, + get_dir_structure, get_file_list) + + +# $PYSSG_HOME is the only env var set +# in the project settings that resemble a path +@pytest.mark.parametrize('path, expected_expanded', [ + ('$PYSSG_HOME', '/tmp/pyssg'), + ('$PYSSG_HOME/', '/tmp/pyssg'), + ('/test$PYSSG_HOME/', '/test/tmp/pyssg'), + ('/test/$PYSSG_HOME/', '/test/tmp/pyssg'), + ('/test/$PYSSG_HOME/test', '/test/tmp/pyssg/test') +]) +def test_path_expansion(path: str, expected_expanded: str) -> None: + expanded: str = get_expanded_path(path) + assert expanded == expected_expanded + + +@pytest.mark.parametrize('path', [ + ('$'), + ('$NON_EXISTENT_VARIABLE'), + ('/path/to/something/$'), + ('/path/to/something/$NON_EXISTENT_VARIABLE') +]) +def test_path_expansion_failure(path: str) -> None: + with pytest.raises(SystemExit) as system_exit: + get_expanded_path(path) + assert system_exit.type == SystemExit + assert system_exit.value.code == 1 + + +def test_checksum(test_dir: str, simple_yaml: str) -> None: + path: str = f'{test_dir}/io_files/{simple_yaml}' + simple_yaml_checksum: str = 'd4f0a3ed56fd530d3ea485dced25534c' + checksum: str = get_checksum(path) + assert checksum == simple_yaml_checksum + + +def test_copy_file(tmp_path: Path, caplog: LogCaptureFixture) -> None: + src: Path = tmp_path/'src' + dst: Path = tmp_path/'dst' + src.mkdir() + dst.mkdir() + src_file: Path = src/'tmp_file.txt' + dst_file: Path = dst/'tmp_file.txt' + src_file.write_text('something') + inf: tuple[str, int, str] = ('pyssg.utils', + INFO, + f'copied file "{src_file}" to "{dst_file}"') + copy_file(str(src_file), str(dst_file)) + assert caplog.record_tuples[-1] == inf + + +def test_copy_file_failure(tmp_path: Path, caplog: LogCaptureFixture) -> None: + src: Path = tmp_path/'src' + dst: Path = tmp_path/'dst' + src.mkdir() + dst.mkdir() + src_file: Path = src/'tmp_file.txt' + dst_file: Path = dst/'tmp_file.txt' + src_file.write_text('something') + dst_file.write_text('something') + inf: tuple[str, int, str] = ('pyssg.utils', + INFO, + f'file "{dst_file}" already exists, ignoring') + copy_file(str(src_file), str(dst_file)) + assert caplog.record_tuples[-1] == inf + + +def test_create_dir(tmp_path: Path, caplog: LogCaptureFixture) -> None: + path: Path = tmp_path/'new_dir' + inf: tuple[str, int, str] = ('pyssg.utils', + INFO, + f'created directory "{path}"') + assert path.exists() is False + create_dir(str(path), False, False) + assert path.exists() is True + assert caplog.record_tuples[-1] == inf + + +def test_create_dir_failure(tmp_path: Path, caplog: LogCaptureFixture) -> None: + path: Path = tmp_path/'new_dir' + inf: tuple[str, int, str] = ('pyssg.utils', + INFO, + f'directory "{path}" exists, ignoring') + path.mkdir() + create_dir(str(path), False, False) + assert caplog.record_tuples[-1] == inf + + +def test_create_dirs(tmp_path: Path, caplog: LogCaptureFixture) -> None: + path: Path = tmp_path/'new_dir' + sub_path: Path = path/'sub_dir' + inf: tuple[str, int, str] = ('pyssg.utils', + INFO, + f'created directory "{sub_path}"') + assert path.exists() is False + assert sub_path.exists() is False + create_dir(str(sub_path), True, False) + assert path.exists() is True + assert sub_path.exists() is True + assert caplog.record_tuples[-1] == inf + + +def test_create_dirs_failure(tmp_path: Path, caplog: LogCaptureFixture) -> None: + path: Path = tmp_path/'new_dir' + sub_path: Path = path/'sub_dir' + inf: tuple[str, int, str] = ('pyssg.utils', + INFO, + f'directory "{sub_path}" exists, ignoring') + path.mkdir() + sub_path.mkdir() + create_dir(str(sub_path), True, False) + assert caplog.record_tuples[-1] == inf + + +@pytest.mark.parametrize('exclude, exp_dir_str', [ + ([], ['second/s1', 'first/f1/f2']), + (['f2'], ['second/s1', 'first/f1']), + (['f1'], ['second/s1', 'first']), + (['second'], ['first/f1/f2']), + (['s1', 'f2'], ['second', 'first/f1']), + (['s1', 'f1'], ['second', 'first']), + (['s1', 'first'], ['second']) +]) +def test_dir_structure(tmp_dir_structure: Path, + exclude: list[str], + exp_dir_str: list[str]) -> None: + dir_str: list[str] = get_dir_structure(str(tmp_dir_structure), exclude) + # order doesn't matter + assert sorted(dir_str) == sorted(exp_dir_str) + + +@pytest.mark.parametrize('exts, exclude_dirs, exp_flist', [ + (('txt',), [], ['f0.txt', 'second/f4.txt', + 'second/s1/f5.txt', 'first/f1.txt', + 'first/f1/f2.txt', 'first/f1/f2/f3.txt']), + (('txt', 'html'), [], ['f0.html', 'f0.txt', + 'second/f4.txt', 'second/f4.html', + 'second/s1/f5.html', 'second/s1/f5.txt', + 'first/f1.html', 'first/f1.txt', + 'first/f1/f2.txt', 'first/f1/f2.html', + 'first/f1/f2/f3.txt', 'first/f1/f2/f3.html']), + (('md',), [], ['f0.md', 'second/f4.md', + 'second/s1/f5.md', 'first/f1.md', + 'first/f1/f2.md', 'first/f1/f2/f3.md']), + (('md',), ['first'], ['f0.md', 'second/f4.md', 'second/s1/f5.md']), + (('md',), ['first', 's1'], ['f0.md', 'second/f4.md']), + (('md',), ['f2', 's1'], ['f0.md', 'second/f4.md', + 'first/f1.md', 'first/f1/f2.md',]), +]) +def test_file_list(tmp_dir_structure: Path, + exts: tuple[str], + exclude_dirs: list[str], + exp_flist: list[str]) -> None: + flist: list[str] = get_file_list(str(tmp_dir_structure), exts, exclude_dirs) + # order doesn't matter + assert sorted(flist) == sorted(exp_flist) diff --git a/tests/test_yaml_parser.py b/tests/test_yaml_parser.py index dd4c6d4..906c7e6 100644 --- a/tests/test_yaml_parser.py +++ b/tests/test_yaml_parser.py @@ -5,7 +5,7 @@ from pyssg.yaml_parser import get_parsed_yaml # and test the join functionality -def test_yaml_resource_read(simple_yaml:str, test_resource: str) -> None: +def test_yaml_resource_read(simple_yaml: str, test_resource: str) -> None: yaml: list[dict[str, Any]] = get_parsed_yaml(simple_yaml, test_resource) assert len(yaml) == 1 @@ -16,7 +16,7 @@ def test_yaml_path_read(test_dir: str) -> None: assert len(yaml) == 1 -def test_yaml_join(simple_yaml:str, test_resource: str) -> None: +def test_yaml_join(simple_yaml: str, test_resource: str) -> None: yaml: dict[str, Any] = get_parsed_yaml(simple_yaml, test_resource)[0] define_str: str = '$PYSSG_HOME/pyssg/site_example/' assert yaml['define'] == define_str -- cgit v1.2.3-54-g00ecf