From cbcf4f6f2c2264cff9e52ebb1cfd654a302d74f0 Mon Sep 17 00:00:00 2001 From: David Luevano Alvarado Date: Wed, 20 Apr 2022 23:08:10 -0600 Subject: minor refactoring --- ChangeLog | 14 +++++++++++ src/pyssg/__init__.py | 8 +++--- src/pyssg/builder.py | 31 +++++++++++------------ src/pyssg/configuration.py | 5 ++-- src/pyssg/database.py | 11 ++++----- src/pyssg/discovery.py | 53 ---------------------------------------- src/pyssg/md_parser.py | 38 +++++++++++++++++++++++----- src/pyssg/page.py | 9 +++---- src/pyssg/per_level_formatter.py | 35 +++++++++++++------------- src/pyssg/pyssg.py | 50 +++++-------------------------------- src/pyssg/utils.py | 53 +++++++++++++++++++++++++++++++++++++--- 11 files changed, 149 insertions(+), 158 deletions(-) delete mode 100644 src/pyssg/discovery.py diff --git a/ChangeLog b/ChangeLog index c96e9bb..aca14d5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,20 @@ CHANGES ======= +v0.6.1 +------ + +* add meaningful error messages when missing mandatory metadata +* add logging to builder, md\_parser and page, and minor code refactor +* minor debug fix for database +* fix db write bug missing newline, minor refactor +* fix pyssg and database errors +* add logging to database, discovery and pyssg +* add typing to formatter +* add debug flag, minor fix in readme +* add initial logging capabilities +* add build and upload command, modified readme + v0.6.0 ------ diff --git a/src/pyssg/__init__.py b/src/pyssg/__init__.py index 3bbfc27..a4e5857 100644 --- a/src/pyssg/__init__.py +++ b/src/pyssg/__init__.py @@ -1,12 +1,12 @@ +from logging import Logger, StreamHandler, getLogger, INFO + from .pyssg import main -import logging -from logging import Logger, StreamHandler from .per_level_formatter import PerLevelFormatter # since this is the root package, setup the logger here -__LOG_LEVEL: int = logging.INFO -log: Logger = logging.getLogger(__name__) +__LOG_LEVEL: int = INFO +log: Logger = getLogger(__name__) log.setLevel(__LOG_LEVEL) ch: StreamHandler = StreamHandler() ch.setLevel(__LOG_LEVEL) diff --git a/src/pyssg/builder.py b/src/pyssg/builder.py index e7a49fe..35502b0 100644 --- a/src/pyssg/builder.py +++ b/src/pyssg/builder.py @@ -1,32 +1,34 @@ import os -import shutil from copy import deepcopy from operator import itemgetter -from jinja2 import Environment, Template -from markdown import Markdown from configparser import ConfigParser -import logging -from logging import Logger +from logging import Logger, getLogger -from .utils import create_dir, copy_file +from jinja2 import Environment, Template, FileSystemLoader as FSLoader + +from .utils import get_file_list, get_dir_structure, create_dir, copy_file from .database import Database from .md_parser import MDParser from .page import Page -from .discovery import get_file_list, get_dir_structure -log: Logger = logging.getLogger(__name__) +log: Logger = getLogger(__name__) class Builder: def __init__(self, config: ConfigParser, - env: Environment, - db: Database, - md: Markdown): + db: Database): log.debug('initializing site builder') self.config: ConfigParser = config - self.env: Environment = env self.db: Database = db - self.md: Markdown = md + + # the autoescape option could be a security risk if used in a dynamic + # website, as far as i can tell + log.debug('initializing the jinja environment') + self.__loader: FSLoader = FSLoader(self.config.get('path', 'plt')) + self.env: Environment = Environment(loader=self.__loader, + autoescape=False, + trim_blocks=True, + lstrip_blocks=True) self.dirs: list[str] = None self.md_files: list[str] = None @@ -55,8 +57,7 @@ class Builder: parser: MDParser = MDParser(self.md_files, self.config, - self.db, - self.md) + self.db) parser.parse_files() # just so i don't have to pass these vars to all the functions diff --git a/src/pyssg/configuration.py b/src/pyssg/configuration.py index a721dba..b193cf8 100644 --- a/src/pyssg/configuration.py +++ b/src/pyssg/configuration.py @@ -3,10 +3,9 @@ from importlib.metadata import version from importlib.resources import path as rpath from datetime import datetime, timezone from configparser import ConfigParser -import logging -from logging import Logger +from logging import Logger, getLogger -log: Logger = logging.getLogger(__name__) +log: Logger = getLogger(__name__) DEFAULT_CONFIG_PATH = '$XDG_CONFIG_HOME/pyssg/config.ini' diff --git a/src/pyssg/database.py b/src/pyssg/database.py index cd4646a..c13df89 100644 --- a/src/pyssg/database.py +++ b/src/pyssg/database.py @@ -1,14 +1,13 @@ import os import sys -import logging -from logging import Logger +from logging import Logger, getLogger -log: Logger = logging.getLogger(__name__) +log: Logger = getLogger(__name__) # db class that works for both html and md files class Database: - COLUMN_NUM: int = 4 + __COLUMN_NUM: int = 4 def __init__(self, db_path: str): log.debug('initializing the page db on path "%s"', db_path) @@ -130,10 +129,10 @@ class Database: r = row.strip() log.debug('row %d content: "%s"', i, r) l = tuple(r.split()) - if len(l) != self.COLUMN_NUM: + if len(l) != self.__COLUMN_NUM: log.critical('row %d doesn\'t contain %s columns,' ' contains %d elements; row %d content: "%s"', - i, self.COLUMN_NUM, len(l), i, r) + i, self.__COLUMN_NUM, len(l), i, r) sys.exit(1) t: list[str] = None diff --git a/src/pyssg/discovery.py b/src/pyssg/discovery.py deleted file mode 100644 index 041ad64..0000000 --- a/src/pyssg/discovery.py +++ /dev/null @@ -1,53 +0,0 @@ -import os -import logging -from logging import Logger - -log: Logger = logging.getLogger(__name__) - - -def get_file_list(path: str, - exts: list[str], - exclude: list[str]=None) -> list[str]: - log.debug('retrieving file list in path "%s" that contain file' - ' extensions (%s) except (%s)', - path, ', '.join(exts), - ', '.join(exclude if exclude is not None else [])) - out: list[str] = [] - for root, dirs, files in os.walk(path): - if exclude is not None: - log.debug('removing excludes from list') - dirs[:] = [d for d in dirs if d not in exclude] - - for f in files: - if f.endswith(tuple(exts)): - stripped_f: str = os.path.join(root, f).replace(path, '')[1:] - out.append(stripped_f) - log.debug('added file "%s" without "%s" part: "%s"', - f, path, stripped_f) - else: - log.debug('ignoring file "%s" as it doesn\'t contain' - ' any of the extensions (%s)', f, ', '.join(exts)) - - return out - - -def get_dir_structure(path: str, - exclude: list[str]=None) -> list[str]: - log.debug('retrieving dir structure in path "%s" except (%s)', - path, ', '.join(exclude if exclude is not None else [])) - out: list[str] = [] - for root, dirs, files in os.walk(path): - if exclude is not None: - log.debug('removing excludes from list') - dirs[:] = [d for d in dirs if d not in exclude] - - for d in dirs: - if root in out: - out.remove(root) - log.debug('removed dir "%s" as it already is in the list', root) - joined_dir: str = os.path.join(root, d) - out.append(joined_dir) - log.debug('added dir "%s" to the list', joined_dir) - - log.debug('removing "%s" from all dirs in list', path) - return [o.replace(path, '')[1:] for o in out] diff --git a/src/pyssg/md_parser.py b/src/pyssg/md_parser.py index b00da19..a516f60 100644 --- a/src/pyssg/md_parser.py +++ b/src/pyssg/md_parser.py @@ -2,27 +2,53 @@ import os from operator import itemgetter from markdown import Markdown from configparser import ConfigParser -import logging -from logging import Logger +from logging import Logger, getLogger + +from markdown import Markdown +from yafg import YafgExtension +from MarkdownHighlight.highlight import HighlightExtension +from markdown_checklist.extension import ChecklistExtension from .database import Database from .page import Page -log: Logger = logging.getLogger(__name__) +log: Logger = getLogger(__name__) + + +def _get_md_obj() -> Markdown: + exts: list = ['extra', + 'meta', + 'sane_lists', + 'smarty', + 'toc', + 'wikilinks', + # stripTitle generates an error when True, + # if there is no title attr + YafgExtension(stripTitle=False, + figureClass="", + figcaptionClass="", + figureNumbering=False, + figureNumberClass="number", + figureNumberText="Figure"), + HighlightExtension(), + ChecklistExtension()] + log.debug('list of md extensions: (%s)', + ', '.join([e if isinstance(e, str) else type(e).__name__ + for e in exts])) + return Markdown(extensions=exts, output_format='html5') # page and file is basically a synonym here... class MDParser: def __init__(self, files: list[str], config: ConfigParser, - db: Database, - md: Markdown): + db: Database): log.debug('initializing the md parser with %d files', len(files)) self.files: list[str] = files self.config: ConfigParser = config self.db: Database = db - self.md: Markdown = md + self.md: Markdown = _get_md_obj() self.all_files: list[Page] = None # updated and modified are synonyms here diff --git a/src/pyssg/page.py b/src/pyssg/page.py index 82a0431..21add82 100644 --- a/src/pyssg/page.py +++ b/src/pyssg/page.py @@ -1,13 +1,10 @@ import os import sys from datetime import datetime, timezone -import logging -from logging import Logger - +from logging import Logger, getLogger from configparser import ConfigParser -from re import L -log: Logger = logging.getLogger(__name__) +log: Logger = getLogger(__name__) class Page: @@ -124,7 +121,7 @@ class Page: self.image_url = \ f'{self.config.get("url", "static")}/{self.meta["image_url"][0]}' except KeyError: - log.debug('using default image, no image_url tag found') + log.debug('using default image, no image_url metadata found') self.image_url = \ f'{self.config.get("url", "static")}/{self.config.get("url", "default_image")}' log.debug('final image url "%s"', self.image_url) diff --git a/src/pyssg/per_level_formatter.py b/src/pyssg/per_level_formatter.py index 2439bd6..7b93890 100644 --- a/src/pyssg/per_level_formatter.py +++ b/src/pyssg/per_level_formatter.py @@ -1,28 +1,27 @@ -import logging -from logging import Formatter +from logging import Formatter, DEBUG, INFO, WARNING, ERROR, CRITICAL -class PerLevelFormatter(logging.Formatter): +class PerLevelFormatter(Formatter): # colors for the terminal in ansi - yellow: str = "\x1b[33m" - red: str = "\x1b[31m" - bold_red: str = "\x1b[31;1m" - reset: str = "\x1b[0m" + __YELLOW: str = "\x1b[33m" + __RED: str = "\x1b[31m" + __BOLD_RED: str = "\x1b[31;1m" + __RESET: str = "\x1b[0m" - DATE_FMT: str = '%Y-%m-%d %H:%M:%S' - COMMON_FMT: str = '[%(levelname)s] [%(module)s:%(funcName)s:%(lineno)d]: %(message)s' - FORMATS: dict[int, str] = { - logging.DEBUG: COMMON_FMT, - logging.INFO: '%(message)s', - logging.WARNING: f'{yellow}{COMMON_FMT}{reset}', - logging.ERROR: f'{red}{COMMON_FMT}{reset}', - logging.CRITICAL: f'{bold_red}{COMMON_FMT}{reset}' + __DATE_FMT: str = '%Y-%m-%d %H:%M:%S' + __COMMON_FMT: str = '[%(levelname)s] [%(module)s:%(funcName)s:%(lineno)d]: %(message)s' + __FORMATS: dict[int, str] = { + DEBUG: __COMMON_FMT, + INFO: '%(message)s', + WARNING: f'{__YELLOW}{__COMMON_FMT}{__RESET}', + ERROR: f'{__RED}{__COMMON_FMT}{__RESET}', + CRITICAL: f'{__BOLD_RED}{__COMMON_FMT}{__RESET}' } def format(self, record: str) -> str: - fmt: str = self.FORMATS.get(record.levelno) - formatter: Formatter = logging.Formatter( - fmt=fmt, datefmt=self.DATE_FMT, style='%') + fmt: str = self.__FORMATS.get(record.levelno) + formatter: Formatter = Formatter( + fmt=fmt, datefmt=self.__DATE_FMT, style='%') return formatter.format(record) diff --git a/src/pyssg/pyssg.py b/src/pyssg/pyssg.py index 2f0730f..931a55f 100644 --- a/src/pyssg/pyssg.py +++ b/src/pyssg/pyssg.py @@ -1,16 +1,9 @@ import os import sys -import logging -from logging import Logger from importlib.resources import path as rpath from typing import Union from configparser import ConfigParser - -from jinja2 import Environment, FileSystemLoader -from markdown import Markdown -from yafg import YafgExtension -from MarkdownHighlight.highlight import HighlightExtension -from markdown_checklist.extension import ChecklistExtension +from logging import Logger, getLogger, DEBUG from .utils import create_dir, copy_file, sanity_check_path from .arg_parser import get_parsed_arguments @@ -18,7 +11,7 @@ from .configuration import get_parsed_config, DEFAULT_CONFIG_PATH, VERSION from .database import Database from .builder import Builder -log: Logger = logging.getLogger(__name__) +log: Logger = getLogger(__name__) def main() -> None: @@ -28,10 +21,10 @@ def main() -> None: # need to modify the root logger specifically, # as it is the one that holds the config # (__name__ happens to resolve to pyssg in __init__) - root_logger: Logger = logging.getLogger('pyssg') - root_logger.setLevel(logging.DEBUG) + root_logger: Logger = getLogger('pyssg') + root_logger.setLevel(DEBUG) for handler in root_logger.handlers: - handler.setLevel(logging.DEBUG) + handler.setLevel(DEBUG) log.debug('changed logging level to DEBUG') if not len(sys.argv) > 1 or (len(sys.argv) == 2 and args['debug']): @@ -86,38 +79,7 @@ def main() -> None: db: Database = Database(os.path.join(config.get('path', 'src'), '.files')) db.read() - # the autoescape option could be a security risk if used in a dynamic - # website, as far as i can tell - log.debug('initializing the jinja environment') - env: Environment = Environment(loader=FileSystemLoader(config.get('path', 'plt')), - autoescape=False, - trim_blocks=True, - lstrip_blocks=True) - - # md extensions - exts: list = ['extra', - 'meta', - 'sane_lists', - 'smarty', - 'toc', - 'wikilinks', - # stripTitle generates an error when True, - # if there is no title attr - YafgExtension(stripTitle=False, - figureClass="", - figcaptionClass="", - figureNumbering=False, - figureNumberClass="number", - figureNumberText="Figure"), - HighlightExtension(), - ChecklistExtension()] - log.debug('list of md extensions: (%s)', - ', '.join([e if isinstance(e, str) else type(e).__name__ - for e in exts])) - log.debug('initializing markdown parser') - md: Markdown = Markdown(extensions=exts, - output_format='html5') - builder: Builder = Builder(config, env, db, md) + builder: Builder = Builder(config, db) builder.build() db.write() diff --git a/src/pyssg/utils.py b/src/pyssg/utils.py index a24d7ca..ffaf8ba 100644 --- a/src/pyssg/utils.py +++ b/src/pyssg/utils.py @@ -1,10 +1,57 @@ import os import sys import shutil -import logging -from logging import Logger +from logging import Logger, getLogger -log: Logger = logging.getLogger(__name__) +log: Logger = getLogger(__name__) + + +def get_file_list(path: str, + exts: list[str], + exclude: list[str]=None) -> list[str]: + log.debug('retrieving file list in path "%s" that contain file' + ' extensions (%s) except (%s)', + path, ', '.join(exts), + ', '.join(exclude if exclude is not None else [])) + out: list[str] = [] + for root, dirs, files in os.walk(path): + if exclude is not None: + log.debug('removing excludes from list') + dirs[:] = [d for d in dirs if d not in exclude] + + for f in files: + if f.endswith(tuple(exts)): + stripped_f: str = os.path.join(root, f).replace(path, '')[1:] + out.append(stripped_f) + log.debug('added file "%s" without "%s" part: "%s"', + f, path, stripped_f) + else: + log.debug('ignoring file "%s" as it doesn\'t contain' + ' any of the extensions (%s)', f, ', '.join(exts)) + + return out + + +def get_dir_structure(path: str, + exclude: list[str]=None) -> list[str]: + log.debug('retrieving dir structure in path "%s" except (%s)', + path, ', '.join(exclude if exclude is not None else [])) + out: list[str] = [] + for root, dirs, files in os.walk(path): + if exclude is not None: + log.debug('removing excludes from list') + dirs[:] = [d for d in dirs if d not in exclude] + + for d in dirs: + if root in out: + out.remove(root) + log.debug('removed dir "%s" as it already is in the list', root) + joined_dir: str = os.path.join(root, d) + out.append(joined_dir) + log.debug('added dir "%s" to the list', joined_dir) + + log.debug('removing "%s" from all dirs in list', path) + return [o.replace(path, '')[1:] for o in out] def create_dir(path: str, p: bool=False) -> None: -- cgit v1.2.3