From 28c2ae9102d4204b3f0a79419eec1e72dbbc529a Mon Sep 17 00:00:00 2001 From: David Luevano Alvarado Date: Tue, 21 Feb 2023 21:02:23 -0600 Subject: add configuration testing, small refactor --- pyproject.toml | 3 +- src/pyssg/configuration.py | 39 ++++++++---- src/pyssg/custom_logger.py | 3 +- src/pyssg/yaml_parser.py | 7 ++- tests/conftest.py | 52 +++++++++++++--- tests/io_files/simple.yaml | 2 +- tests/io_files/simple_missing_dirs.yaml | 21 +++++++ tests/io_files/simple_missing_key.yaml | 29 +++++++++ tests/io_files/simple_missing_root_dir.yaml | 22 +++++++ tests/test_configuration.py | 92 +++++++++++++++++++++++++++++ tests/test_custom_logger.py | 5 +- tests/test_yaml_parser.py | 18 +++--- 12 files changed, 257 insertions(+), 36 deletions(-) create mode 100644 tests/io_files/simple_missing_dirs.yaml create mode 100644 tests/io_files/simple_missing_key.yaml create mode 100644 tests/io_files/simple_missing_root_dir.yaml create mode 100644 tests/test_configuration.py diff --git a/pyproject.toml b/pyproject.toml index 30773d7..3b0018a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,8 @@ testpaths = [ "tests", ] env = [ - "PYMDVAR_TEST_1=1" + "PYMDVAR_TEST_1=1", + "PYSSG_HOME=/tmp/pyssg" ] [tool.mypy] diff --git a/src/pyssg/configuration.py b/src/pyssg/configuration.py index 7b292d5..3cc5430 100644 --- a/src/pyssg/configuration.py +++ b/src/pyssg/configuration.py @@ -2,6 +2,7 @@ import sys from importlib.metadata import version from datetime import datetime, timezone from logging import Logger, getLogger +from typing import Any from .utils import get_expanded_path from .yaml_parser import get_parsed_yaml @@ -23,15 +24,23 @@ def __check_well_formed_config(config: dict, sys.exit(1) # checks for dir_paths if key == 'dirs': + try: + config[key].keys() + except AttributeError: + log.error('config doesn\'t have any dirs configs (dirs.*)') + sys.exit(1) if '/' not in config[key]: - log.error('config doesn\'t have "%s./"', current_key) log.debug('key: %s; config.keys: %s', key, config[key].keys()) + log.error('config doesn\'t have "%s./"', current_key) sys.exit(1) - log.debug('checking "%s" fields for (%s) dir_paths', key, ', '.join(config[key].keys())) + log.debug('checking "%s" fields for (%s) dir_paths', + key, ', '.join(config[key].keys())) for dkey in config[key].keys(): new_current_key: str = f'{current_key}.{dkey}' new_config_base: list[dict] = [config_base[1], config_base[1]] - __check_well_formed_config(config[key][dkey], new_config_base, new_current_key) + __check_well_formed_config(config[key][dkey], + new_config_base, + new_current_key) continue # the case for elements that don't have nested elements if not config_base[0][key]: @@ -48,12 +57,14 @@ def __expand_all_paths(config: dict) -> None: # not necessary to type deeper than the first dict -def get_parsed_config(path: str) -> list[dict]: +def get_parsed_config(path: str, + mc_package: str = 'mandatory_config.yaml', + plt_resource: str = 'pyssg.plt') -> list[dict]: log.debug('reading config file "%s"', path) config_all: list[dict] = get_parsed_yaml(path) - mandatory_config: list[dict] = get_parsed_yaml('mandatory_config.yaml', 'pyssg.plt') - log.info('found %s document(s) for configuration "%s"', len(config_all), path) - log.debug('checking that config file is well formed (at least contains mandatory fields') + mandatory_config: list[dict] = get_parsed_yaml(mc_package, plt_resource) + log.info('found %s document(s) for config "%s"', len(config_all), path) + log.debug('checking that config file is well formed') for config in config_all: __check_well_formed_config(config, mandatory_config) __expand_all_paths(config) @@ -62,11 +73,15 @@ def get_parsed_config(path: str) -> list[dict]: # not necessary to type deeper than the first dict, # static config means config that shouldn't be changed by the user -def get_static_config() -> dict[str, dict]: +def get_static_config(sc_package: str = 'static_config.yaml', + plt_resource: str = 'pyssg.plt') -> dict[str, dict]: log.debug('reading and setting static config') - config: dict = get_parsed_yaml('static_config.yaml', 'pyssg.plt')[0] - # do I really need a lambda function... + config: dict[str, Any] = get_parsed_yaml(sc_package, plt_resource)[0] + + def __time(fmt: str) -> str: + return datetime.now(tz=timezone.utc).strftime(config['fmt'][fmt]) + config['info']['version'] = VERSION - config['info']['rss_run_date'] = datetime.now(tz=timezone.utc).strftime(config['fmt']['rss_date']) - config['info']['sitemap_run_date'] = datetime.now(tz=timezone.utc).strftime(config['fmt']['sitemap_date']) + config['info']['rss_run_date'] = __time('rss_date') + config['info']['sitemap_run_date'] = __time('sitemap_date') return config diff --git a/src/pyssg/custom_logger.py b/src/pyssg/custom_logger.py index 55f3e2d..4eebc4c 100644 --- a/src/pyssg/custom_logger.py +++ b/src/pyssg/custom_logger.py @@ -2,6 +2,7 @@ import sys from logging import (Logger, StreamHandler, Formatter, LogRecord, DEBUG, INFO, WARNING, ERROR, CRITICAL, getLogger) +from typing import TextIO LOG_LEVEL: int = INFO # 'pyssg' es the name of the root logger @@ -38,7 +39,7 @@ class PerLevelFormatter(Formatter): def setup_logger(name: str = LOGGER_NAME, level: int = LOG_LEVEL) -> None: logger: Logger = getLogger(name) - handler: StreamHandler = StreamHandler(sys.stdout) + handler: StreamHandler[TextIO] = StreamHandler(sys.stdout) logger.setLevel(level) handler.setLevel(level) handler.setFormatter(PerLevelFormatter()) diff --git a/src/pyssg/yaml_parser.py b/src/pyssg/yaml_parser.py index 2e1548b..aeb164e 100644 --- a/src/pyssg/yaml_parser.py +++ b/src/pyssg/yaml_parser.py @@ -3,6 +3,7 @@ from yaml import SafeLoader from yaml.nodes import SequenceNode from importlib.resources import path as rpath from logging import Logger, getLogger +from typing import Any log: Logger = getLogger(__name__) @@ -17,15 +18,15 @@ def setup_custom_yaml() -> None: SafeLoader.add_constructor('!join', __join_constructor) -def __read_raw_yaml(path: str) -> list[dict]: - all_docs: list[dict] = [] +def __read_raw_yaml(path: str) -> list[dict[str, Any]]: + all_docs: list[dict[str, Any]] = [] with open(path, 'r') as f: for doc in yaml.safe_load_all(f): all_docs.append(doc) return all_docs -def get_parsed_yaml(resource: str, package: str = '') -> list[dict]: +def get_parsed_yaml(resource: str, package: str = '') -> list[dict[str, Any]]: if package == '': log.debug('parsing yaml; reading "%s"', resource) return __read_raw_yaml(resource) diff --git a/tests/conftest.py b/tests/conftest.py index 1beffc6..b44fbf6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,31 +1,60 @@ import os import sys -from typing import Callable import pytest -from logging import getLogger, DEBUG +from typing import Callable +from pytest import MonkeyPatch +from argparse import ArgumentParser +from datetime import datetime, timezone +from importlib.metadata import version as v +from logging import Logger, getLogger, DEBUG from pyssg.arg_parser import get_parser from pyssg.custom_logger import setup_logger -@pytest.fixture -def test_dir(): - return os.path.dirname(os.path.abspath(__file__)) +@pytest.fixture(scope='session') +def version(): + return v('pyssg') + + +@pytest.fixture(scope='session') +def rss_date_fmt(): + return '%a, %d %b %Y %H:%M:%S GMT' + + +@pytest.fixture(scope='session') +def sitemap_date_fmt(): + return '%Y-%m-%d' + + +@pytest.fixture(scope='session') +def test_dir() -> str: + return str(os.path.dirname(os.path.abspath(__file__))) + + +@pytest.fixture(scope='session') +def test_resource() -> str: + return 'tests.io_files' @pytest.fixture(scope='session') -def arg_parser(): +def simple_yaml() -> str: + return 'simple.yaml' + + +@pytest.fixture(scope='session') +def arg_parser() -> ArgumentParser: return get_parser() @pytest.fixture(scope='session') -def logger(): +def logger() -> Logger: setup_logger(__name__, DEBUG) return getLogger(__name__) @pytest.fixture -def capture_stdout(monkeypatch: Callable) -> dict[str, str | int]: +def capture_stdout(monkeypatch: MonkeyPatch) -> dict[str, str | int]: buffer: dict[str, str | int] = {'stdout': '', 'write_calls': 0} def fake_writer(s): @@ -34,3 +63,10 @@ def capture_stdout(monkeypatch: Callable) -> dict[str, str | int]: monkeypatch.setattr(sys.stdout, 'write', fake_writer) return buffer + + +@pytest.fixture +def get_fmt_time() -> Callable[..., str]: + def fmt_time(fmt: str) -> str: + return datetime.now(tz=timezone.utc).strftime(fmt) + return fmt_time diff --git a/tests/io_files/simple.yaml b/tests/io_files/simple.yaml index 0b722a6..df3888b 100644 --- a/tests/io_files/simple.yaml +++ b/tests/io_files/simple.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -define: &root "$HOME/pyssg/site_example/" +define: &root "$PYSSG_HOME/pyssg/site_example/" title: "Example site" path: diff --git a/tests/io_files/simple_missing_dirs.yaml b/tests/io_files/simple_missing_dirs.yaml new file mode 100644 index 0000000..aa15fb5 --- /dev/null +++ b/tests/io_files/simple_missing_dirs.yaml @@ -0,0 +1,21 @@ +%YAML 1.2 +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" + static: "https://static.example.com" + default_image: "images/default.png" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" + list_date: "%b %d" + list_sep_date: "%B %Y" +dirs: +# test missing dirs +... \ No newline at end of file diff --git a/tests/io_files/simple_missing_key.yaml b/tests/io_files/simple_missing_key.yaml new file mode 100644 index 0000000..ac81563 --- /dev/null +++ b/tests/io_files/simple_missing_key.yaml @@ -0,0 +1,29 @@ +%YAML 1.2 +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +# test missing mandatory key +# title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" + static: "https://static.example.com" + default_image: "images/default.png" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" + list_date: "%b %d" + list_sep_date: "%B %Y" +dirs: + /: + cfg: + plt: "page.html" + tags: False + index: False + rss: False + sitemap: False + exclude_dirs: [] +... \ No newline at end of file diff --git a/tests/io_files/simple_missing_root_dir.yaml b/tests/io_files/simple_missing_root_dir.yaml new file mode 100644 index 0000000..07fa824 --- /dev/null +++ b/tests/io_files/simple_missing_root_dir.yaml @@ -0,0 +1,22 @@ +%YAML 1.2 +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" + static: "https://static.example.com" + default_image: "images/default.png" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" + list_date: "%b %d" + list_sep_date: "%B %Y" +dirs: +# test missing /: + something: +... \ No newline at end of file diff --git a/tests/test_configuration.py b/tests/test_configuration.py new file mode 100644 index 0000000..0db8094 --- /dev/null +++ b/tests/test_configuration.py @@ -0,0 +1,92 @@ +import pytest +from pytest import LogCaptureFixture +from typing import Any, Callable +from logging import DEBUG, INFO, WARNING, ERROR +from pyssg.configuration import get_static_config, get_parsed_config + + +# this test is a bit sketchy, as the way the datetimes are calculated could vary +# by milliseconds or even have a difference in seconds +def test_static_default(rss_date_fmt: str, + sitemap_date_fmt: str, + get_fmt_time: Callable[..., str], + version: str) -> None: + rss_run_date: str = get_fmt_time(rss_date_fmt) + sitemap_run_date: str = get_fmt_time(sitemap_date_fmt) + sc_dict: dict[str, Any] = {'fmt': {'rss_date': rss_date_fmt, + 'sitemap_date': sitemap_date_fmt}, + 'info': {'rss_run_date': rss_run_date, + 'sitemap_run_date': sitemap_run_date, + 'version': version}} + static_config: dict[str, Any] = get_static_config() + assert static_config == sc_dict + + +def test_simple(test_dir: str) -> None: + yaml_dict: dict[str, Any] = {'define': '$PYSSG_HOME/pyssg/site_example/', + 'title': 'Example site', + 'path': { + 'src': '/tmp/pyssg/pyssg/site_example/src', + 'dst': '/tmp/pyssg/pyssg/site_example/dst', + 'plt': '/tmp/pyssg/pyssg/site_example/plt', + 'db': '/tmp/pyssg/pyssg/site_example/.files'}, + 'url': { + 'main': 'https://example.com', + 'static': 'https://static.example.com', + 'default_image': 'images/default.png'}, + 'fmt': { + 'date': '%a, %b %d, %Y @ %H:%M %Z', + 'list_date': '%b %d', + 'list_sep_date': '%B %Y'}, + 'dirs': { + '/': { + 'cfg': { + 'plt': 'page.html', + 'tags': False, + 'index': False, + 'rss': False, + 'sitemap': False, + 'exclude_dirs': []}}}} + yaml_path: str = f'{test_dir}/io_files/simple.yaml' + yaml: list[dict[str, Any]] = get_parsed_config(yaml_path) + assert len(yaml) == 1 + assert yaml[0] == yaml_dict + + +def test_simple_mising_key(test_dir: str, + caplog: LogCaptureFixture) -> None: + err: tuple[str, int, str] = ('pyssg.configuration', + ERROR, + 'config doesn\'t have "title"') + yaml_path: str = f'{test_dir}/io_files/simple_missing_key.yaml' + with pytest.raises(SystemExit) as system_exit: + get_parsed_config(yaml_path) + assert system_exit.type == SystemExit + assert system_exit.value.code == 1 + assert caplog.record_tuples[-1] == err + + +def test_simple_mising_dirs(test_dir: str, + caplog: LogCaptureFixture) -> None: + err: tuple[str, int, str] = ('pyssg.configuration', + ERROR, + 'config doesn\'t have any dirs configs (dirs.*)') + yaml_path: str = f'{test_dir}/io_files/simple_missing_dirs.yaml' + with pytest.raises(SystemExit) as system_exit: + get_parsed_config(yaml_path) + assert system_exit.type == SystemExit + assert system_exit.value.code == 1 + assert caplog.record_tuples[-1] == err + + +def test_simple_root_dir(test_dir: str, + caplog: LogCaptureFixture) -> None: + err: tuple[str, int, str] = ('pyssg.configuration', + ERROR, + 'config doesn\'t have "dirs./"') + yaml_path: str = f'{test_dir}/io_files/simple_missing_root_dir.yaml' + with pytest.raises(SystemExit) as system_exit: + get_parsed_config(yaml_path) + assert system_exit.type == SystemExit + assert system_exit.value.code == 1 + assert caplog.record_tuples[-1] == err diff --git a/tests/test_custom_logger.py b/tests/test_custom_logger.py index 1062e41..9102f52 100644 --- a/tests/test_custom_logger.py +++ b/tests/test_custom_logger.py @@ -1,6 +1,5 @@ import pytest -from logging import DEBUG, INFO, WARNING, ERROR, CRITICAL -from typing import Callable +from logging import Logger, DEBUG, INFO, WARNING, ERROR, CRITICAL @pytest.mark.parametrize('log_level, starts_with, message', [ @@ -13,7 +12,7 @@ from typing import Callable def test_log_levels(log_level: int, starts_with: str, message: str, - logger: Callable, + logger: Logger, capture_stdout: dict[str, str | int]) -> None: logger.log(log_level, message) assert str(capture_stdout['stdout']).startswith(starts_with) diff --git a/tests/test_yaml_parser.py b/tests/test_yaml_parser.py index 9a1b3c3..dd4c6d4 100644 --- a/tests/test_yaml_parser.py +++ b/tests/test_yaml_parser.py @@ -1,19 +1,23 @@ -from importlib.resources import path as rpath +from typing import Any from pyssg.yaml_parser import get_parsed_yaml +# the point of these tests is just to read yaml files +# and test the join functionality -def test_yaml_resource_read() -> None: - yaml: list[dict] = get_parsed_yaml('simple.yaml', 'tests.io_files') + +def test_yaml_resource_read(simple_yaml:str, test_resource: str) -> None: + yaml: list[dict[str, Any]] = get_parsed_yaml(simple_yaml, test_resource) assert len(yaml) == 1 def test_yaml_path_read(test_dir: str) -> None: - yaml: list[dict] = get_parsed_yaml(f'{test_dir}/io_files/simple.yaml') + yaml_path: str = f'{test_dir}/io_files/simple.yaml' + yaml: list[dict[str, Any]] = get_parsed_yaml(yaml_path) assert len(yaml) == 1 -def test_yaml_join() -> None: - yaml: dict = get_parsed_yaml('simple.yaml', 'tests.io_files')[0] - define_str: str = '$HOME/pyssg/site_example/' +def test_yaml_join(simple_yaml:str, test_resource: str) -> None: + yaml: dict[str, Any] = get_parsed_yaml(simple_yaml, test_resource)[0] + define_str: str = '$PYSSG_HOME/pyssg/site_example/' assert yaml['define'] == define_str assert yaml['path']['src'] == f'{define_str}src' -- cgit v1.2.3-70-g09d2