From cbcf4f6f2c2264cff9e52ebb1cfd654a302d74f0 Mon Sep 17 00:00:00 2001
From: David Luevano Alvarado <david@luevano.xyz>
Date: Wed, 20 Apr 2022 23:08:10 -0600
Subject: minor refactoring
---
ChangeLog | 14 +++++++++++
src/pyssg/__init__.py | 8 +++---
src/pyssg/builder.py | 31 +++++++++++------------
src/pyssg/configuration.py | 5 ++--
src/pyssg/database.py | 11 ++++-----
src/pyssg/discovery.py | 53 ----------------------------------------
src/pyssg/md_parser.py | 38 +++++++++++++++++++++++-----
src/pyssg/page.py | 9 +++----
src/pyssg/per_level_formatter.py | 35 +++++++++++++-------------
src/pyssg/pyssg.py | 50 +++++--------------------------------
src/pyssg/utils.py | 53 +++++++++++++++++++++++++++++++++++++---
11 files changed, 149 insertions(+), 158 deletions(-)
delete mode 100644 src/pyssg/discovery.py
diff --git a/ChangeLog b/ChangeLog
index c96e9bb..aca14d5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,20 @@
CHANGES
=======
+v0.6.1
+------
+
+* add meaningful error messages when missing mandatory metadata
+* add logging to builder, md\_parser and page, and minor code refactor
+* minor debug fix for database
+* fix db write bug missing newline, minor refactor
+* fix pyssg and database errors
+* add logging to database, discovery and pyssg
+* add typing to formatter
+* add debug flag, minor fix in readme
+* add initial logging capabilities
+* add build and upload command, modified readme
+
v0.6.0
------
diff --git a/src/pyssg/__init__.py b/src/pyssg/__init__.py
index 3bbfc27..a4e5857 100644
--- a/src/pyssg/__init__.py
+++ b/src/pyssg/__init__.py
@@ -1,12 +1,12 @@
+from logging import Logger, StreamHandler, getLogger, INFO
+
from .pyssg import main
-import logging
-from logging import Logger, StreamHandler
from .per_level_formatter import PerLevelFormatter
# since this is the root package, setup the logger here
-__LOG_LEVEL: int = logging.INFO
-log: Logger = logging.getLogger(__name__)
+__LOG_LEVEL: int = INFO
+log: Logger = getLogger(__name__)
log.setLevel(__LOG_LEVEL)
ch: StreamHandler = StreamHandler()
ch.setLevel(__LOG_LEVEL)
diff --git a/src/pyssg/builder.py b/src/pyssg/builder.py
index e7a49fe..35502b0 100644
--- a/src/pyssg/builder.py
+++ b/src/pyssg/builder.py
@@ -1,32 +1,34 @@
import os
-import shutil
from copy import deepcopy
from operator import itemgetter
-from jinja2 import Environment, Template
-from markdown import Markdown
from configparser import ConfigParser
-import logging
-from logging import Logger
+from logging import Logger, getLogger
-from .utils import create_dir, copy_file
+from jinja2 import Environment, Template, FileSystemLoader as FSLoader
+
+from .utils import get_file_list, get_dir_structure, create_dir, copy_file
from .database import Database
from .md_parser import MDParser
from .page import Page
-from .discovery import get_file_list, get_dir_structure
-log: Logger = logging.getLogger(__name__)
+log: Logger = getLogger(__name__)
class Builder:
def __init__(self, config: ConfigParser,
- env: Environment,
- db: Database,
- md: Markdown):
+ db: Database):
log.debug('initializing site builder')
self.config: ConfigParser = config
- self.env: Environment = env
self.db: Database = db
- self.md: Markdown = md
+
+ # the autoescape option could be a security risk if used in a dynamic
+ # website, as far as i can tell
+ log.debug('initializing the jinja environment')
+ self.__loader: FSLoader = FSLoader(self.config.get('path', 'plt'))
+ self.env: Environment = Environment(loader=self.__loader,
+ autoescape=False,
+ trim_blocks=True,
+ lstrip_blocks=True)
self.dirs: list[str] = None
self.md_files: list[str] = None
@@ -55,8 +57,7 @@ class Builder:
parser: MDParser = MDParser(self.md_files,
self.config,
- self.db,
- self.md)
+ self.db)
parser.parse_files()
# just so i don't have to pass these vars to all the functions
diff --git a/src/pyssg/configuration.py b/src/pyssg/configuration.py
index a721dba..b193cf8 100644
--- a/src/pyssg/configuration.py
+++ b/src/pyssg/configuration.py
@@ -3,10 +3,9 @@ from importlib.metadata import version
from importlib.resources import path as rpath
from datetime import datetime, timezone
from configparser import ConfigParser
-import logging
-from logging import Logger
+from logging import Logger, getLogger
-log: Logger = logging.getLogger(__name__)
+log: Logger = getLogger(__name__)
DEFAULT_CONFIG_PATH = '$XDG_CONFIG_HOME/pyssg/config.ini'
diff --git a/src/pyssg/database.py b/src/pyssg/database.py
index cd4646a..c13df89 100644
--- a/src/pyssg/database.py
+++ b/src/pyssg/database.py
@@ -1,14 +1,13 @@
import os
import sys
-import logging
-from logging import Logger
+from logging import Logger, getLogger
-log: Logger = logging.getLogger(__name__)
+log: Logger = getLogger(__name__)
# db class that works for both html and md files
class Database:
- COLUMN_NUM: int = 4
+ __COLUMN_NUM: int = 4
def __init__(self, db_path: str):
log.debug('initializing the page db on path "%s"', db_path)
@@ -130,10 +129,10 @@ class Database:
r = row.strip()
log.debug('row %d content: "%s"', i, r)
l = tuple(r.split())
- if len(l) != self.COLUMN_NUM:
+ if len(l) != self.__COLUMN_NUM:
log.critical('row %d doesn\'t contain %s columns,'
' contains %d elements; row %d content: "%s"',
- i, self.COLUMN_NUM, len(l), i, r)
+ i, self.__COLUMN_NUM, len(l), i, r)
sys.exit(1)
t: list[str] = None
diff --git a/src/pyssg/discovery.py b/src/pyssg/discovery.py
deleted file mode 100644
index 041ad64..0000000
--- a/src/pyssg/discovery.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import os
-import logging
-from logging import Logger
-
-log: Logger = logging.getLogger(__name__)
-
-
-def get_file_list(path: str,
- exts: list[str],
- exclude: list[str]=None) -> list[str]:
- log.debug('retrieving file list in path "%s" that contain file'
- ' extensions (%s) except (%s)',
- path, ', '.join(exts),
- ', '.join(exclude if exclude is not None else []))
- out: list[str] = []
- for root, dirs, files in os.walk(path):
- if exclude is not None:
- log.debug('removing excludes from list')
- dirs[:] = [d for d in dirs if d not in exclude]
-
- for f in files:
- if f.endswith(tuple(exts)):
- stripped_f: str = os.path.join(root, f).replace(path, '')[1:]
- out.append(stripped_f)
- log.debug('added file "%s" without "%s" part: "%s"',
- f, path, stripped_f)
- else:
- log.debug('ignoring file "%s" as it doesn\'t contain'
- ' any of the extensions (%s)', f, ', '.join(exts))
-
- return out
-
-
-def get_dir_structure(path: str,
- exclude: list[str]=None) -> list[str]:
- log.debug('retrieving dir structure in path "%s" except (%s)',
- path, ', '.join(exclude if exclude is not None else []))
- out: list[str] = []
- for root, dirs, files in os.walk(path):
- if exclude is not None:
- log.debug('removing excludes from list')
- dirs[:] = [d for d in dirs if d not in exclude]
-
- for d in dirs:
- if root in out:
- out.remove(root)
- log.debug('removed dir "%s" as it already is in the list', root)
- joined_dir: str = os.path.join(root, d)
- out.append(joined_dir)
- log.debug('added dir "%s" to the list', joined_dir)
-
- log.debug('removing "%s" from all dirs in list', path)
- return [o.replace(path, '')[1:] for o in out]
diff --git a/src/pyssg/md_parser.py b/src/pyssg/md_parser.py
index b00da19..a516f60 100644
--- a/src/pyssg/md_parser.py
+++ b/src/pyssg/md_parser.py
@@ -2,27 +2,53 @@ import os
from operator import itemgetter
from markdown import Markdown
from configparser import ConfigParser
-import logging
-from logging import Logger
+from logging import Logger, getLogger
+
+from markdown import Markdown
+from yafg import YafgExtension
+from MarkdownHighlight.highlight import HighlightExtension
+from markdown_checklist.extension import ChecklistExtension
from .database import Database
from .page import Page
-log: Logger = logging.getLogger(__name__)
+log: Logger = getLogger(__name__)
+
+
+def _get_md_obj() -> Markdown:
+ exts: list = ['extra',
+ 'meta',
+ 'sane_lists',
+ 'smarty',
+ 'toc',
+ 'wikilinks',
+ # stripTitle generates an error when True,
+ # if there is no title attr
+ YafgExtension(stripTitle=False,
+ figureClass="",
+ figcaptionClass="",
+ figureNumbering=False,
+ figureNumberClass="number",
+ figureNumberText="Figure"),
+ HighlightExtension(),
+ ChecklistExtension()]
+ log.debug('list of md extensions: (%s)',
+ ', '.join([e if isinstance(e, str) else type(e).__name__
+ for e in exts]))
+ return Markdown(extensions=exts, output_format='html5')
# page and file is basically a synonym here...
class MDParser:
def __init__(self, files: list[str],
config: ConfigParser,
- db: Database,
- md: Markdown):
+ db: Database):
log.debug('initializing the md parser with %d files', len(files))
self.files: list[str] = files
self.config: ConfigParser = config
self.db: Database = db
- self.md: Markdown = md
+ self.md: Markdown = _get_md_obj()
self.all_files: list[Page] = None
# updated and modified are synonyms here
diff --git a/src/pyssg/page.py b/src/pyssg/page.py
index 82a0431..21add82 100644
--- a/src/pyssg/page.py
+++ b/src/pyssg/page.py
@@ -1,13 +1,10 @@
import os
import sys
from datetime import datetime, timezone
-import logging
-from logging import Logger
-
+from logging import Logger, getLogger
from configparser import ConfigParser
-from re import L
-log: Logger = logging.getLogger(__name__)
+log: Logger = getLogger(__name__)
class Page:
@@ -124,7 +121,7 @@ class Page:
self.image_url = \
f'{self.config.get("url", "static")}/{self.meta["image_url"][0]}'
except KeyError:
- log.debug('using default image, no image_url tag found')
+ log.debug('using default image, no image_url metadata found')
self.image_url = \
f'{self.config.get("url", "static")}/{self.config.get("url", "default_image")}'
log.debug('final image url "%s"', self.image_url)
diff --git a/src/pyssg/per_level_formatter.py b/src/pyssg/per_level_formatter.py
index 2439bd6..7b93890 100644
--- a/src/pyssg/per_level_formatter.py
+++ b/src/pyssg/per_level_formatter.py
@@ -1,28 +1,27 @@
-import logging
-from logging import Formatter
+from logging import Formatter, DEBUG, INFO, WARNING, ERROR, CRITICAL
-class PerLevelFormatter(logging.Formatter):
+class PerLevelFormatter(Formatter):
# colors for the terminal in ansi
- yellow: str = "\x1b[33m"
- red: str = "\x1b[31m"
- bold_red: str = "\x1b[31;1m"
- reset: str = "\x1b[0m"
+ __YELLOW: str = "\x1b[33m"
+ __RED: str = "\x1b[31m"
+ __BOLD_RED: str = "\x1b[31;1m"
+ __RESET: str = "\x1b[0m"
- DATE_FMT: str = '%Y-%m-%d %H:%M:%S'
- COMMON_FMT: str = '[%(levelname)s] [%(module)s:%(funcName)s:%(lineno)d]: %(message)s'
- FORMATS: dict[int, str] = {
- logging.DEBUG: COMMON_FMT,
- logging.INFO: '%(message)s',
- logging.WARNING: f'{yellow}{COMMON_FMT}{reset}',
- logging.ERROR: f'{red}{COMMON_FMT}{reset}',
- logging.CRITICAL: f'{bold_red}{COMMON_FMT}{reset}'
+ __DATE_FMT: str = '%Y-%m-%d %H:%M:%S'
+ __COMMON_FMT: str = '[%(levelname)s] [%(module)s:%(funcName)s:%(lineno)d]: %(message)s'
+ __FORMATS: dict[int, str] = {
+ DEBUG: __COMMON_FMT,
+ INFO: '%(message)s',
+ WARNING: f'{__YELLOW}{__COMMON_FMT}{__RESET}',
+ ERROR: f'{__RED}{__COMMON_FMT}{__RESET}',
+ CRITICAL: f'{__BOLD_RED}{__COMMON_FMT}{__RESET}'
}
def format(self, record: str) -> str:
- fmt: str = self.FORMATS.get(record.levelno)
- formatter: Formatter = logging.Formatter(
- fmt=fmt, datefmt=self.DATE_FMT, style='%')
+ fmt: str = self.__FORMATS.get(record.levelno)
+ formatter: Formatter = Formatter(
+ fmt=fmt, datefmt=self.__DATE_FMT, style='%')
return formatter.format(record)
diff --git a/src/pyssg/pyssg.py b/src/pyssg/pyssg.py
index 2f0730f..931a55f 100644
--- a/src/pyssg/pyssg.py
+++ b/src/pyssg/pyssg.py
@@ -1,16 +1,9 @@
import os
import sys
-import logging
-from logging import Logger
from importlib.resources import path as rpath
from typing import Union
from configparser import ConfigParser
-
-from jinja2 import Environment, FileSystemLoader
-from markdown import Markdown
-from yafg import YafgExtension
-from MarkdownHighlight.highlight import HighlightExtension
-from markdown_checklist.extension import ChecklistExtension
+from logging import Logger, getLogger, DEBUG
from .utils import create_dir, copy_file, sanity_check_path
from .arg_parser import get_parsed_arguments
@@ -18,7 +11,7 @@ from .configuration import get_parsed_config, DEFAULT_CONFIG_PATH, VERSION
from .database import Database
from .builder import Builder
-log: Logger = logging.getLogger(__name__)
+log: Logger = getLogger(__name__)
def main() -> None:
@@ -28,10 +21,10 @@ def main() -> None:
# need to modify the root logger specifically,
# as it is the one that holds the config
# (__name__ happens to resolve to pyssg in __init__)
- root_logger: Logger = logging.getLogger('pyssg')
- root_logger.setLevel(logging.DEBUG)
+ root_logger: Logger = getLogger('pyssg')
+ root_logger.setLevel(DEBUG)
for handler in root_logger.handlers:
- handler.setLevel(logging.DEBUG)
+ handler.setLevel(DEBUG)
log.debug('changed logging level to DEBUG')
if not len(sys.argv) > 1 or (len(sys.argv) == 2 and args['debug']):
@@ -86,38 +79,7 @@ def main() -> None:
db: Database = Database(os.path.join(config.get('path', 'src'), '.files'))
db.read()
- # the autoescape option could be a security risk if used in a dynamic
- # website, as far as i can tell
- log.debug('initializing the jinja environment')
- env: Environment = Environment(loader=FileSystemLoader(config.get('path', 'plt')),
- autoescape=False,
- trim_blocks=True,
- lstrip_blocks=True)
-
- # md extensions
- exts: list = ['extra',
- 'meta',
- 'sane_lists',
- 'smarty',
- 'toc',
- 'wikilinks',
- # stripTitle generates an error when True,
- # if there is no title attr
- YafgExtension(stripTitle=False,
- figureClass="",
- figcaptionClass="",
- figureNumbering=False,
- figureNumberClass="number",
- figureNumberText="Figure"),
- HighlightExtension(),
- ChecklistExtension()]
- log.debug('list of md extensions: (%s)',
- ', '.join([e if isinstance(e, str) else type(e).__name__
- for e in exts]))
- log.debug('initializing markdown parser')
- md: Markdown = Markdown(extensions=exts,
- output_format='html5')
- builder: Builder = Builder(config, env, db, md)
+ builder: Builder = Builder(config, db)
builder.build()
db.write()
diff --git a/src/pyssg/utils.py b/src/pyssg/utils.py
index a24d7ca..ffaf8ba 100644
--- a/src/pyssg/utils.py
+++ b/src/pyssg/utils.py
@@ -1,10 +1,57 @@
import os
import sys
import shutil
-import logging
-from logging import Logger
+from logging import Logger, getLogger
-log: Logger = logging.getLogger(__name__)
+log: Logger = getLogger(__name__)
+
+
+def get_file_list(path: str,
+ exts: list[str],
+ exclude: list[str]=None) -> list[str]:
+ log.debug('retrieving file list in path "%s" that contain file'
+ ' extensions (%s) except (%s)',
+ path, ', '.join(exts),
+ ', '.join(exclude if exclude is not None else []))
+ out: list[str] = []
+ for root, dirs, files in os.walk(path):
+ if exclude is not None:
+ log.debug('removing excludes from list')
+ dirs[:] = [d for d in dirs if d not in exclude]
+
+ for f in files:
+ if f.endswith(tuple(exts)):
+ stripped_f: str = os.path.join(root, f).replace(path, '')[1:]
+ out.append(stripped_f)
+ log.debug('added file "%s" without "%s" part: "%s"',
+ f, path, stripped_f)
+ else:
+ log.debug('ignoring file "%s" as it doesn\'t contain'
+ ' any of the extensions (%s)', f, ', '.join(exts))
+
+ return out
+
+
+def get_dir_structure(path: str,
+ exclude: list[str]=None) -> list[str]:
+ log.debug('retrieving dir structure in path "%s" except (%s)',
+ path, ', '.join(exclude if exclude is not None else []))
+ out: list[str] = []
+ for root, dirs, files in os.walk(path):
+ if exclude is not None:
+ log.debug('removing excludes from list')
+ dirs[:] = [d for d in dirs if d not in exclude]
+
+ for d in dirs:
+ if root in out:
+ out.remove(root)
+ log.debug('removed dir "%s" as it already is in the list', root)
+ joined_dir: str = os.path.join(root, d)
+ out.append(joined_dir)
+ log.debug('added dir "%s" to the list', joined_dir)
+
+ log.debug('removing "%s" from all dirs in list', path)
+ return [o.replace(path, '')[1:] for o in out]
def create_dir(path: str, p: bool=False) -> None:
--
cgit v1.2.3-70-g09d2