From 35ff363e779149b6364901a53dd1eefe89ca5ddb Mon Sep 17 00:00:00 2001 From: David Luevano Alvarado Date: Thu, 8 Dec 2022 03:18:31 -0600 Subject: refactor code, small bug fixing and changes --- pyssg.xyz/bs | 1 - pyssg.xyz/db.psv | 2 +- pyssg.xyz/live/pyssg.xyz/rss.xml | 9 ++++++++- pyssg.xyz/live/pyssg.xyz/sitemap.xml | 2 +- pyssg.xyz/live/pyssg.xyz/test.html | 9 ++++++++- pyssg.xyz/src/pyssg.xyz/test.md | 11 ++++++++++- src/pyssg/builder.py | 7 ++++--- src/pyssg/configuration.py | 15 +++------------ src/pyssg/database.py | 6 ++---- src/pyssg/database_entry.py | 23 ++++++++++------------- src/pyssg/md_parser.py | 14 ++++++++------ src/pyssg/page.py | 33 ++++++++++++++++----------------- src/pyssg/per_level_formatter.py | 10 ++++------ src/pyssg/pyssg.py | 7 +++++-- 14 files changed, 80 insertions(+), 69 deletions(-) diff --git a/pyssg.xyz/bs b/pyssg.xyz/bs index 1b863dc..1130b71 100755 --- a/pyssg.xyz/bs +++ b/pyssg.xyz/bs @@ -5,6 +5,5 @@ PYSSG_XYZ_DIR=$HOME/pyssg/pyssg.xyz pyssg --config $PYSSG_XYZ_DIR/config.yaml -b --debug -exit 0 echo "rsync: pushing (U) local (pyssg.xyz) to webserver (luevano.xyz). (pyssg.xyz)" rsync -rtuvPL --delete-after --exclude-from=$PYSSG_XYZ_DIR/.rsyncignore $PYSSG_XYZ_DIR/live/ luevanor:/var/www/pyssg.xyz/ \ No newline at end of file diff --git a/pyssg.xyz/db.psv b/pyssg.xyz/db.psv index 3e140d9..c57cde2 100644 --- a/pyssg.xyz/db.psv +++ b/pyssg.xyz/db.psv @@ -1,2 +1,2 @@ -test.md|1670227510.7104242|0.0|00f771e2ad5285488f201809e2b4365e|- +test.md|1670227510.7104242|1670481891.6152065|21ef9b29a3a7b73718b244c9f32ef576|- subdir/test2.md|1670237921.0787709|0.0|309bd695d912634400f1d50b65d51ba3|- diff --git a/pyssg.xyz/live/pyssg.xyz/rss.xml b/pyssg.xyz/live/pyssg.xyz/rss.xml index c6eac30..f55cc2f 100644 --- a/pyssg.xyz/live/pyssg.xyz/rss.xml +++ b/pyssg.xyz/live/pyssg.xyz/rss.xml @@ -36,7 +36,14 @@ https://pyssg.xyz/test.html Mon, 05 Dec 2022 08:05:10 GMT Index page for pyssg.xyz. - This is a small test for the newly added pyssg.xyz.

]]> + This is a small test for the newly added pyssg.xyz.

+ +

subheader 1

+

bold

+

italic

]]> \ No newline at end of file diff --git a/pyssg.xyz/live/pyssg.xyz/sitemap.xml b/pyssg.xyz/live/pyssg.xyz/sitemap.xml index 63949aa..8bc5a06 100644 --- a/pyssg.xyz/live/pyssg.xyz/sitemap.xml +++ b/pyssg.xyz/live/pyssg.xyz/sitemap.xml @@ -10,7 +10,7 @@ https://pyssg.xyz/test.html - 2022-12-05 + 2022-12-08 weekly 1.0 diff --git a/pyssg.xyz/live/pyssg.xyz/test.html b/pyssg.xyz/live/pyssg.xyz/test.html index 0ce6abd..f847273 100644 --- a/pyssg.xyz/live/pyssg.xyz/test.html +++ b/pyssg.xyz/live/pyssg.xyz/test.html @@ -9,9 +9,16 @@

Index

By David Luevano

Created: Mon, Dec 05, 2022 @ 08:05 UTC

-

Modified:

+

Modified: Thu, Dec 08, 2022 @ 06:44 UTC

This is a small test for the newly added pyssg.xyz.

+ +

subheader 1

+

bold

+

italic

Tags:

diff --git a/pyssg.xyz/src/pyssg.xyz/test.md b/pyssg.xyz/src/pyssg.xyz/test.md index 8ff6cea..b95cb6f 100644 --- a/pyssg.xyz/src/pyssg.xyz/test.md +++ b/pyssg.xyz/src/pyssg.xyz/test.md @@ -3,4 +3,13 @@ author: David Luevano lang: en summary: Index page for pyssg.xyz. -This is a small test for the newly added pyssg.xyz. \ No newline at end of file +This is a small test for the newly added pyssg.xyz. + +- item 1 +- item 2 + +## subheader 1 + +**bold** + +*italic* \ No newline at end of file diff --git a/src/pyssg/builder.py b/src/pyssg/builder.py index 65c5837..f7537d6 100644 --- a/src/pyssg/builder.py +++ b/src/pyssg/builder.py @@ -58,6 +58,7 @@ class Builder: self.html_files: list[str] # files and pages are synoyms + # TODO: include updated_tags when when implemented self.all_files: list[Page] self.updated_files: list[Page] self.all_tags: list[tuple[str, str]] @@ -115,10 +116,10 @@ class Builder: self.__render_tags('tag.html') default_plts: dict[str, str] = {'index': 'index.html', - 'rss': 'rss.xml', - 'sitemap': 'sitemap.xml'} + 'rss': 'rss.xml', + 'sitemap': 'sitemap.xml'} for opt in default_plts.keys(): - if opt in self.dir_cfg: + if self.dir_cfg[opt]: if isinstance(self.dir_cfg[opt], str): self.__render_template(self.dir_cfg[opt], default_plts[opt], diff --git a/src/pyssg/configuration.py b/src/pyssg/configuration.py index a2b48b6..258729b 100644 --- a/src/pyssg/configuration.py +++ b/src/pyssg/configuration.py @@ -22,21 +22,18 @@ def __check_well_formed_config(config: dict, log.error('config doesn\'t have "%s"', current_key) log.debug('key: %s; config.keys: %s', key, config.keys()) sys.exit(1) - # checks for dir_paths if key == 'dirs': if '/' not in config[key]: log.error('config doesn\'t have "%s./"', current_key) log.debug('key: %s; config.keys: %s', key, config[key].keys()) sys.exit(1) - log.debug('checking "%s" fields for (%s) dir_paths', key, ', '.join(config[key].keys())) for dkey in config[key].keys(): new_current_key: str = f'{current_key}.{dkey}' new_config_base: list[dict] = [config_base[1], config_base[1]] __check_well_formed_config(config[key][dkey], new_config_base, new_current_key) continue - # the case for elements that don't have nested elements if not config_base[0][key]: log.debug('"%s" doesn\'t need nested elements', current_key) @@ -56,15 +53,11 @@ def get_parsed_config(path: str) -> list[dict]: log.debug('reading config file "%s"', path) config: list[dict] = get_parsed_yaml(path) mandatory_config: list[dict] = get_parsed_yaml('mandatory_config.yaml', 'pyssg.plt') - log.info('found %s document(s) for configuration "%s"', len(config), path) log.debug('checking that config file is well formed (at least contains mandatory fields') # TODO: make it work with n yaml docs __check_well_formed_config(config[0], mandatory_config) - log.error('testing') - sys.exit(1) __expand_all_paths(config[0]) - return config @@ -74,10 +67,8 @@ def get_static_config() -> dict[str, dict]: log.debug('reading and setting static config') config: dict = get_parsed_yaml('static_config.yaml', 'pyssg.plt')[0] # do I really need a lambda function... - current_time = lambda x : datetime.now(tz=timezone.utc).strftime(x) - + time = lambda x : datetime.now(tz=timezone.utc).strftime(config['fmt'][x]) config['info']['version'] = VERSION - config['info']['rss_run_date'] = current_time(config['fmt']['rss_date']) - config['info']['sitemap_run_date'] = current_time(config['fmt']['sitemap_date']) - + config['info']['rss_run_date'] = time('rss_date') + config['info']['sitemap_run_date'] = time('sitemap_date') return config diff --git a/src/pyssg/database.py b/src/pyssg/database.py index d4b6a86..d9c6467 100644 --- a/src/pyssg/database.py +++ b/src/pyssg/database.py @@ -21,8 +21,10 @@ class Database: # updates the tags for a specific entry (file) # file_name only contains the entry name (not an absolute path) + # TODO: make the function return true/false if updated def update_tags(self, file_name: str, new_tags: list[str]) -> None: + # technically, I should ensure this function can only run if self.e is populated if file_name in self.e: log.debug('updating tags for entry "%s"', file_name) log.debug('entry "%s" old content: %s', @@ -103,7 +105,6 @@ class Database: if not os.path.isfile(self.db_path): log.error('"%s" is not a file"', self.db_path) sys.exit(1) - return True def _get_csv_rows(self) -> list[list[str]]: @@ -112,7 +113,6 @@ class Database: csv_reader = csv.reader(f, delimiter=self.__COLUMN_DELIMITER) rows = list(csv_reader) log.debug('db contains %d rows', len(rows)) - return rows def read(self) -> None: @@ -127,12 +127,10 @@ class Database: i: int = it + 1 col_num: int = len(row) log.debug('row %d content: "%s"', i, row) - if col_num != self.__COLUMN_NUM: log.critical('row %d doesn\'t contain %s columns, contains %d' ' columns: "%s"', i, self.__COLUMN_NUM, col_num, row) sys.exit(1) - entry: DatabaseEntry = DatabaseEntry(row) self.e[entry.fname] = entry diff --git a/src/pyssg/database_entry.py b/src/pyssg/database_entry.py index 3fec92a..90c3f5f 100644 --- a/src/pyssg/database_entry.py +++ b/src/pyssg/database_entry.py @@ -12,29 +12,26 @@ class DatabaseEntry: self.ctimestamp: float = float(entry[1]) self.mtimestamp: float = float(entry[2]) self.checksum: str = entry[3] - self.tags: list[str] + self.tags: list[str] = [] if isinstance(entry[4], list): self.tags = entry[4] else: - if entry[4] == '-': - self.tags = [] - else: + if entry[4] != '-': self.tags = entry[4].split(',') - log.debug('tag content: [%s]', ', '.join(self.tags)) - + log.debug('"%s" tag: [%s]', self.fname, ', '.join(self.tags)) def __str__(self) -> str: _return_str: str = '[{}, {}, {}, {}, [{}]]'\ - .format(self.fname, - self.ctimestamp, - self.mtimestamp, - self.checksum, - ', '.join(self.tags)) + .format(self.fname, + self.ctimestamp, + self.mtimestamp, + self.checksum, + ', '.join(self.tags)) return _return_str - + # used for csv writing def get_raw_entry(self) -> list[str]: return [self.fname, str(self.ctimestamp), @@ -42,6 +39,6 @@ class DatabaseEntry: self.checksum, ','.join(self.tags) if self.tags else '-'] - + # TODO: make the function return true/false if updated def update_tags(self, new_tags: list[str]) -> None: self.tags = new_tags diff --git a/src/pyssg/md_parser.py b/src/pyssg/md_parser.py index 8c61bc5..14544a3 100644 --- a/src/pyssg/md_parser.py +++ b/src/pyssg/md_parser.py @@ -35,12 +35,12 @@ def _get_md_obj() -> Markdown: log.debug('list of md extensions: (%s)', ', '.join([e if isinstance(e, str) else type(e).__name__ for e in exts])) - # for some reason, the d efinition for output_format doesn't include html5 + # for some reason, the definition for output_format doesn't include html5 # even though it is listed in the documentation, ignoring return Markdown(extensions=exts, output_format='html5') # type: ignore -# page and file is basically a synonym here... +# page and file is basically a synonym class MDParser: def __init__(self, files: list[str], config: dict, @@ -48,18 +48,17 @@ class MDParser: db: Database): log.debug('initializing the md parser with %d files', len(files)) self.files: list[str] = files - self.config: dict = config self.dir_config: dict = dir_config self.db: Database = db self.md: Markdown = _get_md_obj() + # TODO: include updated_tags when when implemented self.all_files: list[Page] = [] # updated and modified are synonyms here self.updated_files: list[Page] = [] self.all_tags: list[tuple[str, str]] = [] - def parse_files(self) -> None: log.debug('parsing all files') for f in self.files: @@ -67,6 +66,7 @@ class MDParser: src_file: str = os.path.join(self.dir_config['src'], f) log.debug('path "%s"', src_file) # get flag if update is successful + # update is only used to get a separate list of only updated files file_updated: bool = self.db.update(src_file, remove=f'{self.dir_config["src"]}/') log.debug('parsing md into html') @@ -89,6 +89,8 @@ class MDParser: self.all_files.append(page) # parse tags + # TODO: only parse tags if set in config + # TODO: separate all tags and only updated tags if page.tags is not None: log.debug('parsing tags') # add its tag to corresponding db entry if existent @@ -97,10 +99,10 @@ class MDParser: log.debug('add all tags to tag list') for t in page.tags: if t[0] not in list(map(itemgetter(0), self.all_tags)): - log.debug('adding tag "%s" as it\'s not present in tag list', t[0]) + log.debug('adding tag "%s"', t[0]) self.all_tags.append(t) else: - log.debug('ignoring tag "%s" as it\'s present in tag list', t[0]) + log.debug('ignoring tag "%s"; already present', t[0]) else: log.debug('no tags to parse') diff --git a/src/pyssg/page.py b/src/pyssg/page.py index c77e3fa..2a6fce7 100644 --- a/src/pyssg/page.py +++ b/src/pyssg/page.py @@ -58,11 +58,9 @@ class Page: # also from self.meta, but for og metadata self.og: dict[str, str] = dict() - def __lt__(self, other): return self.ctimestamp < other.ctimestamp - def __get_mandatory_meta(self, meta: str) -> str: try: log.debug('parsing required metadata "%s"', meta) @@ -72,7 +70,6 @@ class Page: meta, os.path.join(self.dir_config['src'], self.name)) sys.exit(1) - # parses meta from self.meta, for og, it prioritizes, # the actual og meta def parse_metadata(self): @@ -85,23 +82,24 @@ class Page: log.debug('parsing timestamp') self.cdatetime = datetime.fromtimestamp(self.ctimestamp, tz=timezone.utc) - self.cdate = self.cdatetime.strftime(self.config['fmt']['date']) - self.cdate_list = self.cdatetime.strftime(self.config['fmt']['list_date']) - self.cdate_list_sep = self.cdatetime.strftime(self.config['fmt']['list_sep_date']) - self.cdate_rss = self.cdatetime.strftime(self.config['fmt']['rss_date']) - self.cdate_sitemap = \ - self.cdatetime.strftime(self.config['fmt']['sitemap_date']) + # these could be actual function + cdate = lambda x : self.cdatetime.strftime(self.config['fmt'][x]) + mdate = lambda x : self.mdatetime.strftime(self.config['fmt'][x]) + + self.cdate = cdate('date') + self.cdate_list = cdate('list_date') + self.cdate_list_sep = cdate('list_sep_date') + self.cdate_rss = cdate('rss_date') + self.cdate_sitemap = cdate('sitemap_date') if self.mtimestamp != 0.0: log.debug('parsing modified timestamp') - self.mdatetime = datetime.fromtimestamp(self.mtimestamp, - tz=timezone.utc) - self.mdate = self.mdatetime.strftime(self.config['fmt']['date']) - self.mdate_list = self.mdatetime.strftime(self.config['fmt']['list_date']) - self.mdate_list_sep = self.mdatetime.strftime(self.config['fmt']['list_sep_date']) - self.mdate_rss = self.mdatetime.strftime(self.config['fmt']['rss_date']) - self.mdate_sitemap = \ - self.mdatetime.strftime(self.config['fmt']['sitemap_date']) + self.mdatetime = datetime.fromtimestamp(self.mtimestamp, tz=timezone.utc) + self.mdate = mdate('date') + self.mdate_list = mdate('list_date') + self.mdate_list_sep = mdate('list_sep_date') + self.mdate_rss = mdate('rss_date') + self.mdate_sitemap = mdate('sitemap_date') else: log.debug('not parsing modified timestamp, hasn\'t been modified') @@ -145,6 +143,7 @@ class Page: ' "default_image" set in the config file') # if contains open graph elements + # TODO: better handle thsi part try: # og_e = object graph entry og_elements: list[str] = self.meta['og'] diff --git a/src/pyssg/per_level_formatter.py b/src/pyssg/per_level_formatter.py index 04f943b..394471e 100644 --- a/src/pyssg/per_level_formatter.py +++ b/src/pyssg/per_level_formatter.py @@ -4,10 +4,10 @@ from logging import Formatter, LogRecord, DEBUG, INFO, WARNING, ERROR, CRITICAL # and everything else with more info and with colors class PerLevelFormatter(Formatter): # colors for the terminal in ansi - __YELLOW: str = "\x1b[33m" - __RED: str = "\x1b[31m" - __BOLD_RED: str = "\x1b[31;1m" - __RESET: str = "\x1b[0m" + __YELLOW: str = '\x1b[33m' + __RED: str = '\x1b[31m' + __BOLD_RED: str = '\x1b[31;1m' + __RESET: str = '\x1b[0m' __DATE_FMT: str = '%Y-%m-%d %H:%M:%S' __COMMON_FMT: str = '[%(levelname)s] [%(module)s:%(funcName)s:%(lineno)d]: %(message)s' @@ -19,12 +19,10 @@ class PerLevelFormatter(Formatter): CRITICAL: f'{__BOLD_RED}{__COMMON_FMT}{__RESET}' } - def format(self, record: LogRecord) -> str: # this should never fail, as __FORMATS is defined above, # so no issue of just converting to str fmt: str = str(self.__FORMATS.get(record.levelno)) formatter: Formatter = Formatter( fmt=fmt, datefmt=self.__DATE_FMT, style='%') - return formatter.format(record) diff --git a/src/pyssg/pyssg.py b/src/pyssg/pyssg.py index 718e043..94584aa 100644 --- a/src/pyssg/pyssg.py +++ b/src/pyssg/pyssg.py @@ -107,8 +107,11 @@ def main() -> None: db: Database = Database(config['path']['db']) db.read() - builder: Builder = Builder(config, db, "/") - builder.build() + log.debug('building all dir_paths found in config') + for dir_path in config['dirs'].keys(): + log.debug('building for "%s"', dir_path) + builder: Builder = Builder(config, db, dir_path) + builder.build() db.write() log.info('finished building the html files') -- cgit v1.2.3-54-g00ecf