From 35ff363e779149b6364901a53dd1eefe89ca5ddb Mon Sep 17 00:00:00 2001
From: David Luevano Alvarado <david@luevano.xyz>
Date: Thu, 8 Dec 2022 03:18:31 -0600
Subject: refactor code, small bug fixing and changes
---
pyssg.xyz/bs | 1 -
pyssg.xyz/db.psv | 2 +-
pyssg.xyz/live/pyssg.xyz/rss.xml | 9 ++++++++-
pyssg.xyz/live/pyssg.xyz/sitemap.xml | 2 +-
pyssg.xyz/live/pyssg.xyz/test.html | 9 ++++++++-
pyssg.xyz/src/pyssg.xyz/test.md | 11 ++++++++++-
src/pyssg/builder.py | 7 ++++---
src/pyssg/configuration.py | 15 +++------------
src/pyssg/database.py | 6 ++----
src/pyssg/database_entry.py | 23 ++++++++++-------------
src/pyssg/md_parser.py | 14 ++++++++------
src/pyssg/page.py | 33 ++++++++++++++++-----------------
src/pyssg/per_level_formatter.py | 10 ++++------
src/pyssg/pyssg.py | 7 +++++--
14 files changed, 80 insertions(+), 69 deletions(-)
diff --git a/pyssg.xyz/bs b/pyssg.xyz/bs
index 1b863dc..1130b71 100755
--- a/pyssg.xyz/bs
+++ b/pyssg.xyz/bs
@@ -5,6 +5,5 @@ PYSSG_XYZ_DIR=$HOME/pyssg/pyssg.xyz
pyssg --config $PYSSG_XYZ_DIR/config.yaml -b --debug
-exit 0
echo "rsync: pushing (U) local (pyssg.xyz) to webserver (luevano.xyz). (pyssg.xyz)"
rsync -rtuvPL --delete-after --exclude-from=$PYSSG_XYZ_DIR/.rsyncignore $PYSSG_XYZ_DIR/live/ luevanor:/var/www/pyssg.xyz/
\ No newline at end of file
diff --git a/pyssg.xyz/db.psv b/pyssg.xyz/db.psv
index 3e140d9..c57cde2 100644
--- a/pyssg.xyz/db.psv
+++ b/pyssg.xyz/db.psv
@@ -1,2 +1,2 @@
-test.md|1670227510.7104242|0.0|00f771e2ad5285488f201809e2b4365e|-
+test.md|1670227510.7104242|1670481891.6152065|21ef9b29a3a7b73718b244c9f32ef576|-
subdir/test2.md|1670237921.0787709|0.0|309bd695d912634400f1d50b65d51ba3|-
diff --git a/pyssg.xyz/live/pyssg.xyz/rss.xml b/pyssg.xyz/live/pyssg.xyz/rss.xml
index c6eac30..f55cc2f 100644
--- a/pyssg.xyz/live/pyssg.xyz/rss.xml
+++ b/pyssg.xyz/live/pyssg.xyz/rss.xml
@@ -36,7 +36,14 @@
<guid isPermaLink="true">https://pyssg.xyz/test.html</guid>
<pubDate>Mon, 05 Dec 2022 08:05:10 GMT</pubDate>
<description>Index page for pyssg.xyz.</description>
- <content:encoded><![CDATA[<p>This is a small test for the newly added pyssg.xyz.</p>]]></content:encoded>
+ <content:encoded><![CDATA[<p>This is a small test for the newly added pyssg.xyz.</p>
+<ul>
+<li>item 1</li>
+<li>item 2</li>
+</ul>
+<h2 id="subheader-1">subheader 1</h2>
+<p><strong>bold</strong></p>
+<p><em>italic</em></p>]]></content:encoded>
</item>
</channel>
</rss>
\ No newline at end of file
diff --git a/pyssg.xyz/live/pyssg.xyz/sitemap.xml b/pyssg.xyz/live/pyssg.xyz/sitemap.xml
index 63949aa..8bc5a06 100644
--- a/pyssg.xyz/live/pyssg.xyz/sitemap.xml
+++ b/pyssg.xyz/live/pyssg.xyz/sitemap.xml
@@ -10,7 +10,7 @@
</url>
<url>
<loc>https://pyssg.xyz/test.html</loc>
- <lastmod>2022-12-05</lastmod>
+ <lastmod>2022-12-08</lastmod>
<changefreq>weekly</changefreq>
<priority>1.0</priority>
</url>
diff --git a/pyssg.xyz/live/pyssg.xyz/test.html b/pyssg.xyz/live/pyssg.xyz/test.html
index 0ce6abd..f847273 100644
--- a/pyssg.xyz/live/pyssg.xyz/test.html
+++ b/pyssg.xyz/live/pyssg.xyz/test.html
@@ -9,9 +9,16 @@
<h1>Index</h1>
<p>By David Luevano</p>
<p>Created: Mon, Dec 05, 2022 @ 08:05 UTC</p>
- <p>Modified: </p>
+ <p>Modified: Thu, Dec 08, 2022 @ 06:44 UTC</p>
<p>This is a small test for the newly added pyssg.xyz.</p>
+<ul>
+<li>item 1</li>
+<li>item 2</li>
+</ul>
+<h2 id="subheader-1">subheader 1</h2>
+<p><strong>bold</strong></p>
+<p><em>italic</em></p>
<p>Tags:
</p>
diff --git a/pyssg.xyz/src/pyssg.xyz/test.md b/pyssg.xyz/src/pyssg.xyz/test.md
index 8ff6cea..b95cb6f 100644
--- a/pyssg.xyz/src/pyssg.xyz/test.md
+++ b/pyssg.xyz/src/pyssg.xyz/test.md
@@ -3,4 +3,13 @@ author: David Luevano
lang: en
summary: Index page for pyssg.xyz.
-This is a small test for the newly added pyssg.xyz.
\ No newline at end of file
+This is a small test for the newly added pyssg.xyz.
+
+- item 1
+- item 2
+
+## subheader 1
+
+**bold**
+
+*italic*
\ No newline at end of file
diff --git a/src/pyssg/builder.py b/src/pyssg/builder.py
index 65c5837..f7537d6 100644
--- a/src/pyssg/builder.py
+++ b/src/pyssg/builder.py
@@ -58,6 +58,7 @@ class Builder:
self.html_files: list[str]
# files and pages are synoyms
+ # TODO: include updated_tags when when implemented
self.all_files: list[Page]
self.updated_files: list[Page]
self.all_tags: list[tuple[str, str]]
@@ -115,10 +116,10 @@ class Builder:
self.__render_tags('tag.html')
default_plts: dict[str, str] = {'index': 'index.html',
- 'rss': 'rss.xml',
- 'sitemap': 'sitemap.xml'}
+ 'rss': 'rss.xml',
+ 'sitemap': 'sitemap.xml'}
for opt in default_plts.keys():
- if opt in self.dir_cfg:
+ if self.dir_cfg[opt]:
if isinstance(self.dir_cfg[opt], str):
self.__render_template(self.dir_cfg[opt],
default_plts[opt],
diff --git a/src/pyssg/configuration.py b/src/pyssg/configuration.py
index a2b48b6..258729b 100644
--- a/src/pyssg/configuration.py
+++ b/src/pyssg/configuration.py
@@ -22,21 +22,18 @@ def __check_well_formed_config(config: dict,
log.error('config doesn\'t have "%s"', current_key)
log.debug('key: %s; config.keys: %s', key, config.keys())
sys.exit(1)
-
# checks for dir_paths
if key == 'dirs':
if '/' not in config[key]:
log.error('config doesn\'t have "%s./"', current_key)
log.debug('key: %s; config.keys: %s', key, config[key].keys())
sys.exit(1)
-
log.debug('checking "%s" fields for (%s) dir_paths', key, ', '.join(config[key].keys()))
for dkey in config[key].keys():
new_current_key: str = f'{current_key}.{dkey}'
new_config_base: list[dict] = [config_base[1], config_base[1]]
__check_well_formed_config(config[key][dkey], new_config_base, new_current_key)
continue
-
# the case for elements that don't have nested elements
if not config_base[0][key]:
log.debug('"%s" doesn\'t need nested elements', current_key)
@@ -56,15 +53,11 @@ def get_parsed_config(path: str) -> list[dict]:
log.debug('reading config file "%s"', path)
config: list[dict] = get_parsed_yaml(path)
mandatory_config: list[dict] = get_parsed_yaml('mandatory_config.yaml', 'pyssg.plt')
-
log.info('found %s document(s) for configuration "%s"', len(config), path)
log.debug('checking that config file is well formed (at least contains mandatory fields')
# TODO: make it work with n yaml docs
__check_well_formed_config(config[0], mandatory_config)
- log.error('testing')
- sys.exit(1)
__expand_all_paths(config[0])
-
return config
@@ -74,10 +67,8 @@ def get_static_config() -> dict[str, dict]:
log.debug('reading and setting static config')
config: dict = get_parsed_yaml('static_config.yaml', 'pyssg.plt')[0]
# do I really need a lambda function...
- current_time = lambda x : datetime.now(tz=timezone.utc).strftime(x)
-
+ time = lambda x : datetime.now(tz=timezone.utc).strftime(config['fmt'][x])
config['info']['version'] = VERSION
- config['info']['rss_run_date'] = current_time(config['fmt']['rss_date'])
- config['info']['sitemap_run_date'] = current_time(config['fmt']['sitemap_date'])
-
+ config['info']['rss_run_date'] = time('rss_date')
+ config['info']['sitemap_run_date'] = time('sitemap_date')
return config
diff --git a/src/pyssg/database.py b/src/pyssg/database.py
index d4b6a86..d9c6467 100644
--- a/src/pyssg/database.py
+++ b/src/pyssg/database.py
@@ -21,8 +21,10 @@ class Database:
# updates the tags for a specific entry (file)
# file_name only contains the entry name (not an absolute path)
+ # TODO: make the function return true/false if updated
def update_tags(self, file_name: str,
new_tags: list[str]) -> None:
+ # technically, I should ensure this function can only run if self.e is populated
if file_name in self.e:
log.debug('updating tags for entry "%s"', file_name)
log.debug('entry "%s" old content: %s',
@@ -103,7 +105,6 @@ class Database:
if not os.path.isfile(self.db_path):
log.error('"%s" is not a file"', self.db_path)
sys.exit(1)
-
return True
def _get_csv_rows(self) -> list[list[str]]:
@@ -112,7 +113,6 @@ class Database:
csv_reader = csv.reader(f, delimiter=self.__COLUMN_DELIMITER)
rows = list(csv_reader)
log.debug('db contains %d rows', len(rows))
-
return rows
def read(self) -> None:
@@ -127,12 +127,10 @@ class Database:
i: int = it + 1
col_num: int = len(row)
log.debug('row %d content: "%s"', i, row)
-
if col_num != self.__COLUMN_NUM:
log.critical('row %d doesn\'t contain %s columns, contains %d'
' columns: "%s"',
i, self.__COLUMN_NUM, col_num, row)
sys.exit(1)
-
entry: DatabaseEntry = DatabaseEntry(row)
self.e[entry.fname] = entry
diff --git a/src/pyssg/database_entry.py b/src/pyssg/database_entry.py
index 3fec92a..90c3f5f 100644
--- a/src/pyssg/database_entry.py
+++ b/src/pyssg/database_entry.py
@@ -12,29 +12,26 @@ class DatabaseEntry:
self.ctimestamp: float = float(entry[1])
self.mtimestamp: float = float(entry[2])
self.checksum: str = entry[3]
- self.tags: list[str]
+ self.tags: list[str] = []
if isinstance(entry[4], list):
self.tags = entry[4]
else:
- if entry[4] == '-':
- self.tags = []
- else:
+ if entry[4] != '-':
self.tags = entry[4].split(',')
- log.debug('tag content: [%s]', ', '.join(self.tags))
-
+ log.debug('"%s" tag: [%s]', self.fname, ', '.join(self.tags))
def __str__(self) -> str:
_return_str: str = '[{}, {}, {}, {}, [{}]]'\
- .format(self.fname,
- self.ctimestamp,
- self.mtimestamp,
- self.checksum,
- ', '.join(self.tags))
+ .format(self.fname,
+ self.ctimestamp,
+ self.mtimestamp,
+ self.checksum,
+ ', '.join(self.tags))
return _return_str
-
+ # used for csv writing
def get_raw_entry(self) -> list[str]:
return [self.fname,
str(self.ctimestamp),
@@ -42,6 +39,6 @@ class DatabaseEntry:
self.checksum,
','.join(self.tags) if self.tags else '-']
-
+ # TODO: make the function return true/false if updated
def update_tags(self, new_tags: list[str]) -> None:
self.tags = new_tags
diff --git a/src/pyssg/md_parser.py b/src/pyssg/md_parser.py
index 8c61bc5..14544a3 100644
--- a/src/pyssg/md_parser.py
+++ b/src/pyssg/md_parser.py
@@ -35,12 +35,12 @@ def _get_md_obj() -> Markdown:
log.debug('list of md extensions: (%s)',
', '.join([e if isinstance(e, str) else type(e).__name__
for e in exts]))
- # for some reason, the d efinition for output_format doesn't include html5
+ # for some reason, the definition for output_format doesn't include html5
# even though it is listed in the documentation, ignoring
return Markdown(extensions=exts, output_format='html5') # type: ignore
-# page and file is basically a synonym here...
+# page and file is basically a synonym
class MDParser:
def __init__(self, files: list[str],
config: dict,
@@ -48,18 +48,17 @@ class MDParser:
db: Database):
log.debug('initializing the md parser with %d files', len(files))
self.files: list[str] = files
-
self.config: dict = config
self.dir_config: dict = dir_config
self.db: Database = db
self.md: Markdown = _get_md_obj()
+ # TODO: include updated_tags when when implemented
self.all_files: list[Page] = []
# updated and modified are synonyms here
self.updated_files: list[Page] = []
self.all_tags: list[tuple[str, str]] = []
-
def parse_files(self) -> None:
log.debug('parsing all files')
for f in self.files:
@@ -67,6 +66,7 @@ class MDParser:
src_file: str = os.path.join(self.dir_config['src'], f)
log.debug('path "%s"', src_file)
# get flag if update is successful
+ # update is only used to get a separate list of only updated files
file_updated: bool = self.db.update(src_file, remove=f'{self.dir_config["src"]}/')
log.debug('parsing md into html')
@@ -89,6 +89,8 @@ class MDParser:
self.all_files.append(page)
# parse tags
+ # TODO: only parse tags if set in config
+ # TODO: separate all tags and only updated tags
if page.tags is not None:
log.debug('parsing tags')
# add its tag to corresponding db entry if existent
@@ -97,10 +99,10 @@ class MDParser:
log.debug('add all tags to tag list')
for t in page.tags:
if t[0] not in list(map(itemgetter(0), self.all_tags)):
- log.debug('adding tag "%s" as it\'s not present in tag list', t[0])
+ log.debug('adding tag "%s"', t[0])
self.all_tags.append(t)
else:
- log.debug('ignoring tag "%s" as it\'s present in tag list', t[0])
+ log.debug('ignoring tag "%s"; already present', t[0])
else:
log.debug('no tags to parse')
diff --git a/src/pyssg/page.py b/src/pyssg/page.py
index c77e3fa..2a6fce7 100644
--- a/src/pyssg/page.py
+++ b/src/pyssg/page.py
@@ -58,11 +58,9 @@ class Page:
# also from self.meta, but for og metadata
self.og: dict[str, str] = dict()
-
def __lt__(self, other):
return self.ctimestamp < other.ctimestamp
-
def __get_mandatory_meta(self, meta: str) -> str:
try:
log.debug('parsing required metadata "%s"', meta)
@@ -72,7 +70,6 @@ class Page:
meta, os.path.join(self.dir_config['src'], self.name))
sys.exit(1)
-
# parses meta from self.meta, for og, it prioritizes,
# the actual og meta
def parse_metadata(self):
@@ -85,23 +82,24 @@ class Page:
log.debug('parsing timestamp')
self.cdatetime = datetime.fromtimestamp(self.ctimestamp,
tz=timezone.utc)
- self.cdate = self.cdatetime.strftime(self.config['fmt']['date'])
- self.cdate_list = self.cdatetime.strftime(self.config['fmt']['list_date'])
- self.cdate_list_sep = self.cdatetime.strftime(self.config['fmt']['list_sep_date'])
- self.cdate_rss = self.cdatetime.strftime(self.config['fmt']['rss_date'])
- self.cdate_sitemap = \
- self.cdatetime.strftime(self.config['fmt']['sitemap_date'])
+ # these could be actual function
+ cdate = lambda x : self.cdatetime.strftime(self.config['fmt'][x])
+ mdate = lambda x : self.mdatetime.strftime(self.config['fmt'][x])
+
+ self.cdate = cdate('date')
+ self.cdate_list = cdate('list_date')
+ self.cdate_list_sep = cdate('list_sep_date')
+ self.cdate_rss = cdate('rss_date')
+ self.cdate_sitemap = cdate('sitemap_date')
if self.mtimestamp != 0.0:
log.debug('parsing modified timestamp')
- self.mdatetime = datetime.fromtimestamp(self.mtimestamp,
- tz=timezone.utc)
- self.mdate = self.mdatetime.strftime(self.config['fmt']['date'])
- self.mdate_list = self.mdatetime.strftime(self.config['fmt']['list_date'])
- self.mdate_list_sep = self.mdatetime.strftime(self.config['fmt']['list_sep_date'])
- self.mdate_rss = self.mdatetime.strftime(self.config['fmt']['rss_date'])
- self.mdate_sitemap = \
- self.mdatetime.strftime(self.config['fmt']['sitemap_date'])
+ self.mdatetime = datetime.fromtimestamp(self.mtimestamp, tz=timezone.utc)
+ self.mdate = mdate('date')
+ self.mdate_list = mdate('list_date')
+ self.mdate_list_sep = mdate('list_sep_date')
+ self.mdate_rss = mdate('rss_date')
+ self.mdate_sitemap = mdate('sitemap_date')
else:
log.debug('not parsing modified timestamp, hasn\'t been modified')
@@ -145,6 +143,7 @@ class Page:
' "default_image" set in the config file')
# if contains open graph elements
+ # TODO: better handle thsi part
try:
# og_e = object graph entry
og_elements: list[str] = self.meta['og']
diff --git a/src/pyssg/per_level_formatter.py b/src/pyssg/per_level_formatter.py
index 04f943b..394471e 100644
--- a/src/pyssg/per_level_formatter.py
+++ b/src/pyssg/per_level_formatter.py
@@ -4,10 +4,10 @@ from logging import Formatter, LogRecord, DEBUG, INFO, WARNING, ERROR, CRITICAL
# and everything else with more info and with colors
class PerLevelFormatter(Formatter):
# colors for the terminal in ansi
- __YELLOW: str = "\x1b[33m"
- __RED: str = "\x1b[31m"
- __BOLD_RED: str = "\x1b[31;1m"
- __RESET: str = "\x1b[0m"
+ __YELLOW: str = '\x1b[33m'
+ __RED: str = '\x1b[31m'
+ __BOLD_RED: str = '\x1b[31;1m'
+ __RESET: str = '\x1b[0m'
__DATE_FMT: str = '%Y-%m-%d %H:%M:%S'
__COMMON_FMT: str = '[%(levelname)s] [%(module)s:%(funcName)s:%(lineno)d]: %(message)s'
@@ -19,12 +19,10 @@ class PerLevelFormatter(Formatter):
CRITICAL: f'{__BOLD_RED}{__COMMON_FMT}{__RESET}'
}
-
def format(self, record: LogRecord) -> str:
# this should never fail, as __FORMATS is defined above,
# so no issue of just converting to str
fmt: str = str(self.__FORMATS.get(record.levelno))
formatter: Formatter = Formatter(
fmt=fmt, datefmt=self.__DATE_FMT, style='%')
-
return formatter.format(record)
diff --git a/src/pyssg/pyssg.py b/src/pyssg/pyssg.py
index 718e043..94584aa 100644
--- a/src/pyssg/pyssg.py
+++ b/src/pyssg/pyssg.py
@@ -107,8 +107,11 @@ def main() -> None:
db: Database = Database(config['path']['db'])
db.read()
- builder: Builder = Builder(config, db, "/")
- builder.build()
+ log.debug('building all dir_paths found in config')
+ for dir_path in config['dirs'].keys():
+ log.debug('building for "%s"', dir_path)
+ builder: Builder = Builder(config, db, dir_path)
+ builder.build()
db.write()
log.info('finished building the html files')
--
cgit v1.2.3-70-g09d2