From 4bcc2029c25cb5b640eff5ea59ab020da6a42e2b Mon Sep 17 00:00:00 2001 From: David Luevano Alvarado Date: Tue, 22 Aug 2023 01:05:45 -0600 Subject: refactor: simplify config parsing and init got rid of all the overengineered configuration parsing, everything is more simple and the error handling is left to the user as it would only mean checking their configuration based on the error message --- src/pyssg/arg_parser.py | 19 ++----- src/pyssg/configuration.py | 107 ++++++++++++------------------------ src/pyssg/plt/default.yaml | 28 ++++++---- src/pyssg/plt/entry.md | 6 ++ src/pyssg/plt/mandatory_config.yaml | 23 -------- src/pyssg/plt/static_config.yaml | 8 --- src/pyssg/pyssg.py | 86 +++++++++++++---------------- src/pyssg/utils.py | 6 ++ src/pyssg/yaml_parser.py | 10 +--- 9 files changed, 107 insertions(+), 186 deletions(-) create mode 100644 src/pyssg/plt/entry.md delete mode 100644 src/pyssg/plt/mandatory_config.yaml delete mode 100644 src/pyssg/plt/static_config.yaml diff --git a/src/pyssg/arg_parser.py b/src/pyssg/arg_parser.py index 5b66697..17176f0 100644 --- a/src/pyssg/arg_parser.py +++ b/src/pyssg/arg_parser.py @@ -11,25 +11,18 @@ def get_parser() -> ArgumentParser: action='store_true', help='''print program version''') parser.add_argument('-c', '--config', - # don't give a default here, as it would seem like - # --config was passed - # default='$XDG_CONFIG_HOME/pyssg/config.ini', type=str, - help='''config file (path) to read from; if not passed, - '$XDG_CONFIG_HOME/pyssg/config.yaml' is used''') - parser.add_argument('--copy-default-config', - action='store_true', - help='''copies the default config to path specified in - --config flag''') - parser.add_argument('-i', '--init', - action='store_true', - help='''initializes the directory structures and copies - over default templates''') + help='''config file path; if not passed, './config.yaml' + is assumed''') parser.add_argument('-b', '--build', action='store_true', help='''generates all HTML files by parsing MD files present in source directory and copies over manually written HTML files''') + parser.add_argument('-i', '--init', + type=str, + help='''initializes the directory structures and copies + default templates''') parser.add_argument('--debug', action='store_true', help='''change logging level from info to debug''') diff --git a/src/pyssg/configuration.py b/src/pyssg/configuration.py index c7b4248..7ac83f8 100644 --- a/src/pyssg/configuration.py +++ b/src/pyssg/configuration.py @@ -1,57 +1,16 @@ +import os import sys from importlib.metadata import version -from datetime import datetime, timezone from logging import Logger, getLogger from typing import Any -from .utils import get_expanded_path -from .yaml_parser import get_parsed_yaml +from .utils import get_expanded_path, get_time_now +from .yaml_parser import get_yaml log: Logger = getLogger(__name__) -DEFAULT_CONFIG_PATH: str = '$XDG_CONFIG_HOME/pyssg/config.yaml' VERSION: str = version('pyssg') -# TODO: add checking for extensions config (such as pymdvar) -def __check_well_formed_config(config: dict[str, Any], - config_base: list[dict[str, Any]], - prefix_key: str = '') -> None: - for key in config_base[0].keys(): - new_config_base: list[dict[str, Any]] = [] - current_key: str = f'{prefix_key}.{key}' if prefix_key != '' else key - log.debug('checking "%s"', current_key) - if key not in config: - log.error('config doesn\'t have "%s"', current_key) - log.debug('key: %s; config.keys: %s', key, config.keys()) - sys.exit(1) - # checks for dir_paths - if key == 'dirs': - try: - config[key].keys() - except AttributeError: - log.error('config doesn\'t have any dirs (dirs.*)') - sys.exit(1) - if '/' not in config[key]: - log.debug('key: %s; config.keys: %s', key, config[key].keys()) - log.error('config doesn\'t have "%s./"', current_key) - sys.exit(1) - log.debug('checking "%s" fields for (%s) dir_paths', - key, ', '.join(config[key].keys())) - for dkey in config[key].keys(): - new_current_key: str = f'{current_key}.{dkey}' - new_config_base = [config_base[1], config_base[1]] - __check_well_formed_config(config[key][dkey], - new_config_base, - new_current_key) - continue - # the case for elements that don't have nested elements - if not config_base[0][key]: - log.debug('"%s" doesn\'t need nested elements', current_key) - continue - new_config_base = [config_base[0][key], config_base[1]] - __check_well_formed_config(config[key], new_config_base, current_key) - - def __expand_all_paths(config: dict[str, Any]) -> None: log.debug('expanding all path options: %s', config['path'].keys()) for option in config['path'].keys(): @@ -59,33 +18,35 @@ def __expand_all_paths(config: dict[str, Any]) -> None: # not necessary to type deeper than the first dict -def get_parsed_config(path: str, - mc_package: str = 'mandatory_config.yaml', - plt_resource: str = 'pyssg.plt') -> list[dict[str, Any]]: - log.debug('reading config file "%s"', path) - config_all: list[dict[str, Any]] = get_parsed_yaml(path) - mandatory_config: list[dict[str, Any]] = get_parsed_yaml(mc_package, - plt_resource) - log.info('found %s document(s) for config "%s"', len(config_all), path) - log.debug('checking that config file is well formed') - for config in config_all: - __check_well_formed_config(config, mandatory_config) - __expand_all_paths(config) - return config_all - - -# not necessary to type deeper than the first dict, -# static config means config that shouldn't be changed by the user -def get_static_config(sc_package: str = 'static_config.yaml', - plt_resource: str = 'pyssg.plt') -> dict[str, Any]: - log.debug('reading and setting static config') - config: dict[str, Any] = get_parsed_yaml(sc_package, plt_resource)[0] - - # TODO: move this to utils and update the tests - def __time(fmt: str) -> str: - return datetime.now(tz=timezone.utc).strftime(config['fmt'][fmt]) - - config['info']['version'] = VERSION - config['info']['rss_run_date'] = __time('rss_date') - config['info']['sitemap_run_date'] = __time('sitemap_date') +def get_parsed_config(path: str) -> list[dict[str, Any]]: + log.debug('reading default config') + config: list[dict[str, Any]] = get_yaml(path) + log.info('found %s document(s) for config "%s"', len(config), path) + + if len(config) < 2: + log.error('config file requires at least 2 documents:' + ' main config and root dir config') + sys.exit(1) + + __expand_all_paths(config[0]) + + log.debug('adding possible missing configuration and populating') + if 'fmt' not in config[0]: + config[0]['fmt'] = dict() + if 'rss_date' not in config[0]['fmt']: + config[0]['fmt']['rss_date'] = '%a, %d %b %Y %H:%M:%S GMT' + if 'sitemap_date' not in config[0]['fmt']: + config[0]['fmt']['sitemap_date'] = '%Y-%m-%d' + + if 'info' not in config[0]: + config[0]['info'] = dict() + config[0]['info']['version'] = VERSION + config[0]['info']['rss_run_date'] = get_time_now('rss_date') + config[0]['info']['sitemap_run_date'] = get_time_now('sitemap_date') + + if config[1]['dir'] != "/": + log.error('the first directory config needs to be' + ' root (/), found %s instead', config[1]['dir']) + sys.exit(1) return config + diff --git a/src/pyssg/plt/default.yaml b/src/pyssg/plt/default.yaml index ca2f7ad..b458b75 100644 --- a/src/pyssg/plt/default.yaml +++ b/src/pyssg/plt/default.yaml @@ -1,23 +1,27 @@ %YAML 1.2 --- -define: &root "$HOME/pyssg/site_example/" +define: &root "./" title: "Example site" path: src: !join [*root, "src"] dst: !join [*root, "dst"] plt: !join [*root, "plt"] - db: !join [*root, ".files"] + db: !join [*root, "db"] url: - main: "https://example.com" + base: "https://example.com" fmt: date: "%a, %b %d, %Y @ %H:%M %Z" -dirs: - /: - cfg: - plt: "page.html" - tags: False - index: False - rss: False - sitemap: False -... \ No newline at end of file + rss_date: "%a, %d %b %Y %H:%M:%S GMT" + sitemap_date: "%Y-%m-%d" +info: + version: "0.0.0" +... +--- +dir: "/" +plt: "page.html" +tags: False +index: False +rss: False +sitemap: False +... diff --git a/src/pyssg/plt/entry.md b/src/pyssg/plt/entry.md new file mode 100644 index 0000000..c5d1c10 --- /dev/null +++ b/src/pyssg/plt/entry.md @@ -0,0 +1,6 @@ +title: Sample entry +author: David L +tags: sample + english + +This is a sample entry. diff --git a/src/pyssg/plt/mandatory_config.yaml b/src/pyssg/plt/mandatory_config.yaml deleted file mode 100644 index 4b4acac..0000000 --- a/src/pyssg/plt/mandatory_config.yaml +++ /dev/null @@ -1,23 +0,0 @@ -%YAML 1.2 ---- -title: -path: - src: - dst: - plt: - db: -url: - main: -fmt: - date: -dirs: - /: -... ---- -cfg: - plt: - tags: - index: - rss: - sitemap: -... \ No newline at end of file diff --git a/src/pyssg/plt/static_config.yaml b/src/pyssg/plt/static_config.yaml deleted file mode 100644 index 745c767..0000000 --- a/src/pyssg/plt/static_config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -%YAML 1.2 ---- -fmt: - rss_date: "%a, %d %b %Y %H:%M:%S GMT" - sitemap_date: "%Y-%m-%d" -info: - version: "0.0.0" -... \ No newline at end of file diff --git a/src/pyssg/pyssg.py b/src/pyssg/pyssg.py index 9b46a66..e266747 100644 --- a/src/pyssg/pyssg.py +++ b/src/pyssg/pyssg.py @@ -7,7 +7,7 @@ from argparse import ArgumentParser from .arg_parser import get_parser from .utils import create_dir, copy_file, get_expanded_path -from .configuration import get_parsed_config, get_static_config, DEFAULT_CONFIG_PATH, VERSION +from .configuration import get_parsed_config, VERSION from .database import Database from .builder import Builder @@ -18,6 +18,7 @@ def main() -> None: arg_parser: ArgumentParser = get_parser() args: dict[str, Union[str, bool]] = vars(arg_parser.parse_args()) + # TODO: move this logic to the logger # too messy to place at utils.py, don't want to be # passing the arg parser around def _log_perror(message: str) -> None: @@ -40,6 +41,7 @@ def main() -> None: log.info('pyssg v%s', VERSION) sys.exit(0) + # TODO: move this logic to the logger if args['debug']: # need to modify the root logger specifically, # as it is the one that holds the config @@ -50,68 +52,56 @@ def main() -> None: handler.setLevel(DEBUG) log.debug('changed logging level to DEBUG') - config_path: str = str(args['config']) if args['config'] else DEFAULT_CONFIG_PATH - # only needed for the DEFAULT_CONFIG_PATH - config_path = get_expanded_path(config_path) - config_dir, _ = os.path.split(config_path) - log.debug('checked config file path, final config path "%s"', config_path) - - if args['copy_default_config']: - log.info('copying default config file') - create_dir(config_dir) + # TODO: modify init arg in argparser + if args['init']: + init_dir: str = os.path.normpath(get_expanded_path(str(args['init']))) + log.info('initializing directory structure and copying templates') + create_dir(init_dir) with rpath('pyssg.plt', 'default.yaml') as p: - copy_file(str(p), config_path) + copy_file(str(p), os.path.join(init_dir, 'config.yaml')) + create_dir(os.path.join(init_dir, 'src')) + create_dir(os.path.join(init_dir, 'dst')) + create_dir(os.path.join(init_dir, 'plt')) + files: list[str] = ['index.html', + 'page.html', + 'tag.html', + 'rss.xml', + 'sitemap.xml', + 'entry.md'] + log.debug('list of files to copy over: (%s)', ', '.join(files)) + for f in files: + plt_file: str = os.path.join(os.path.join(init_dir, 'plt'), f) + with rpath('pyssg.plt', f) as p: + copy_file(str(p), plt_file) + log.info('finished initialization') sys.exit(0) + config_path: str = get_expanded_path(str((args['config']))) \ + if args['config'] else 'config.yaml' + if not os.path.exists(config_path): log.error('config file does\'t exist in path "%s"; make sure' - ' the path is correct; use --copy-default-config if it\'s the' + ' the path is correct; use --init if it\'s the' ' first time if you haven\'t already', config_path) sys.exit(1) log.debug('reading config files') - config_all: list[dict] = get_parsed_config(config_path) - static_config: dict = get_static_config() - log.debug('applying static_config for each config document') - for config in config_all: - config['fmt']['rss_date'] = static_config['fmt']['rss_date'] - config['fmt']['sitemap_date'] = static_config['fmt']['sitemap_date'] - config['info'] = dict() - config['info']['version'] = static_config['info']['version'] - config['info']['debug'] = str(args['debug']) - - if args['init']: - log.info('initializing the directory structure and copying over templates') - for config in config_all: - log.info('initializing directories for "%s"', config['title']) - create_dir(config['path']['src']) - # dst gets created on builder - # create_dir(config['path']['dst']) - create_dir(config['path']['plt']) - files: list[str] = ['index.html', - 'page.html', - 'tag.html', - 'rss.xml', - 'sitemap.xml'] - log.debug('list of files to copy over: (%s)', ', '.join(files)) - for f in files: - plt_file: str = os.path.join(config['path']['plt'], f) - with rpath('pyssg.plt', f) as p: - copy_file(str(p), plt_file) - log.info('finished initialization') - sys.exit(0) + config: list[dict] = get_parsed_config(config_path) + print(config) + log.debug('exiting due to testing') + sys.exit(0) if args['build']: log.info('building the html files') - for config in config_all: - log.info('building html for "%s"', config['title']) - db: Database = Database(config['path']['db']) + for conf in config: + log.info('building html for "%s"', conf['title']) + db: Database = Database(conf['path']['db']) db.read() - log.debug('building all dir_paths found in config') - for dir_path in config['dirs'].keys(): + log.debug('building all dir_paths found in conf') + for dir_path in conf['dirs'].keys(): log.debug('building for "%s"', dir_path) - builder: Builder = Builder(config, db, dir_path) + builder: Builder = Builder(conf, db, dir_path) builder.build() db.write() diff --git a/src/pyssg/utils.py b/src/pyssg/utils.py index b1ed8c1..d391ccf 100644 --- a/src/pyssg/utils.py +++ b/src/pyssg/utils.py @@ -3,6 +3,7 @@ import sys import shutil from hashlib import md5 from logging import Logger, getLogger +from datetime import datetime, timezone log: Logger = getLogger(__name__) @@ -106,3 +107,8 @@ def get_expanded_path(path: str) -> str: sys.exit(1) log.debug('expanded path "%s" to "%s"', path, expanded_path) return expanded_path + + +def get_time_now(fmt: str, tz: timezone=timezone.utc) -> str: + return datetime.now(tz=tz).strftime(fmt) + diff --git a/src/pyssg/yaml_parser.py b/src/pyssg/yaml_parser.py index aeb164e..227dc63 100644 --- a/src/pyssg/yaml_parser.py +++ b/src/pyssg/yaml_parser.py @@ -18,18 +18,10 @@ def setup_custom_yaml() -> None: SafeLoader.add_constructor('!join', __join_constructor) -def __read_raw_yaml(path: str) -> list[dict[str, Any]]: +def get_yaml(path: str) -> list[dict[str, Any]]: all_docs: list[dict[str, Any]] = [] with open(path, 'r') as f: for doc in yaml.safe_load_all(f): all_docs.append(doc) return all_docs - -def get_parsed_yaml(resource: str, package: str = '') -> list[dict[str, Any]]: - if package == '': - log.debug('parsing yaml; reading "%s"', resource) - return __read_raw_yaml(resource) - log.debug('parsing yaml; reading "%s.%s"', package, resource) - with rpath(package, resource) as p: - return __read_raw_yaml(str(p)) -- cgit v1.2.3-54-g00ecf