summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDavid Luevano Alvarado <david@luevano.xyz>2022-12-07 03:31:51 -0600
committerDavid Luevano Alvarado <david@luevano.xyz>2022-12-07 03:31:51 -0600
commit40d23848d816816b3db3e7296e8a28f964b53786 (patch)
treed3f27d599b4087d91f76fef62b7449f5cf6be2f6
parent0b8441c79b047f81526bbb83febc40d7530e35d6 (diff)
refactor code, fix config checker and new files for pyssg.xyz
Diffstat
-rw-r--r--pyssg.xyz/.rsyncignore10
-rwxr-xr-xpyssg.xyz/bs10
-rw-r--r--pyssg.xyz/config.yaml25
-rw-r--r--pyssg.xyz/live/pyssg.xyz/index.html22
-rw-r--r--pyssg.xyz/live/pyssg.xyz/rss.xml42
-rw-r--r--pyssg.xyz/live/pyssg.xyz/sitemap.xml18
-rw-r--r--pyssg.xyz/live/pyssg.xyz/subdir/test2.html (renamed from pyssg.xyz/dst/subdir/test2.html)0
-rw-r--r--pyssg.xyz/live/pyssg.xyz/test.html (renamed from pyssg.xyz/dst/test.html)0
-rwxr-xr-xpyssg.xyz/pyssg_alias.sh5
-rw-r--r--pyssg.xyz/src/pyssg.xyz/subdir/test2.md (renamed from pyssg.xyz/src/subdir/test2.md)0
-rw-r--r--pyssg.xyz/src/pyssg.xyz/test.md (renamed from pyssg.xyz/src/test.md)0
-rw-r--r--src/pyssg/builder.py164
-rw-r--r--src/pyssg/configuration.py63
-rw-r--r--src/pyssg/database.py6
-rw-r--r--src/pyssg/plt/default.yaml13
-rw-r--r--src/pyssg/plt/mandatory_config.yaml9
-rw-r--r--src/pyssg/utils.py63
-rw-r--r--src/pyssg/yaml_parser.py31
18 files changed, 296 insertions, 185 deletions
diff --git a/pyssg.xyz/.rsyncignore b/pyssg.xyz/.rsyncignore
new file mode 100644
index 0000000..0a0d740
--- /dev/null
+++ b/pyssg.xyz/.rsyncignore
@@ -0,0 +1,10 @@
+# List of directories and files to ignore
+# Git specific
+.gitignore
+README.md
+LICENSE
+.git/
+
+# Rsync specific
+.rsyncignore
+.venv/ \ No newline at end of file
diff --git a/pyssg.xyz/bs b/pyssg.xyz/bs
new file mode 100755
index 0000000..1b863dc
--- /dev/null
+++ b/pyssg.xyz/bs
@@ -0,0 +1,10 @@
+#!/bin/sh
+# bs = build and sync, to pyssg.xyz
+
+PYSSG_XYZ_DIR=$HOME/pyssg/pyssg.xyz
+
+pyssg --config $PYSSG_XYZ_DIR/config.yaml -b --debug
+
+exit 0
+echo "rsync: pushing (U) local (pyssg.xyz) to webserver (luevano.xyz). (pyssg.xyz)"
+rsync -rtuvPL --delete-after --exclude-from=$PYSSG_XYZ_DIR/.rsyncignore $PYSSG_XYZ_DIR/live/ luevanor:/var/www/pyssg.xyz/ \ No newline at end of file
diff --git a/pyssg.xyz/config.yaml b/pyssg.xyz/config.yaml
index 976ab4f..71e8a1c 100644
--- a/pyssg.xyz/config.yaml
+++ b/pyssg.xyz/config.yaml
@@ -4,8 +4,8 @@ define: &root_path "$HOME/pyssg/pyssg.xyz/"
title: "pyssg"
path:
- src: !join [*root_path, "src"]
- dst: !join [*root_path, "dst"]
+ src: !join [*root_path, "src/pyssg.xyz"]
+ dst: !join [*root_path, "live/pyssg.xyz"]
plt: !join [*root_path, "plt"]
db: !join [*root_path, "db.psv"]
url:
@@ -18,13 +18,18 @@ fmt:
list_sep_date: "%B %Y"
dirs:
/:
- plt: "page.html"
- tags: False
- index: False
- rss: False
- sitemap: False
- exclude_dirs: []
+ cfg:
+ plt: "page.html"
+ tags: False
+ index: False
+ rss: False
+ sitemap: False
+ exclude_dirs: []
articles:
- plt: "page.html"
- tags: True
+ cfg:
+ plt: "page.html"
+ tags: True
+ index: False
+ rss: False
+ sitemap: False
... \ No newline at end of file
diff --git a/pyssg.xyz/live/pyssg.xyz/index.html b/pyssg.xyz/live/pyssg.xyz/index.html
new file mode 100644
index 0000000..723cbd1
--- /dev/null
+++ b/pyssg.xyz/live/pyssg.xyz/index.html
@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <base href="https://static.pyssg.xyz">
+ <title>Index -- pyssg</title>
+ </head>
+ <body>
+ <h1>Index -- pyssg</h1>
+ <p>Some text here.</p>
+
+ <p>Tags:
+ </p>
+
+ <h2>Articles</h2>
+ <ul>
+ <h3>December 2022</h3>
+ <li>Dec 05 - <a href="https://pyssg.xyz/subdir/test2.html">Test file in subdir</a></li>
+ <li>Dec 05 - <a href="https://pyssg.xyz/test.html">Index</a></li>
+ </ul>
+ </body>
+</html> \ No newline at end of file
diff --git a/pyssg.xyz/live/pyssg.xyz/rss.xml b/pyssg.xyz/live/pyssg.xyz/rss.xml
new file mode 100644
index 0000000..c6eac30
--- /dev/null
+++ b/pyssg.xyz/live/pyssg.xyz/rss.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<rss version="2.0"
+ xmlns:atom="http://www.w3.org/2005/Atom"
+ xmlns:content="http://purl.org/rss/1.0/modules/content/">
+ <channel>
+ <title>pyssg</title>
+ <link>https://pyssg.xyz</link>
+ <atom:link href="https://pyssg.xyz/rss.xml" rel="self" type="application/rss+xml"/>
+ <description>Short site description.</description>
+ <language>en-us</language>
+ <category>Blog</category>
+ <copyright>Copyright 2021 Somebody</copyright>
+ <managingEditor>some@one.com (Sombody)</managingEditor>
+ <webMaster>some@one.com (Sombody)</webMaster>
+ <pubDate></pubDate>
+ <lastBuildDate></lastBuildDate>
+ <generator>pyssg v0.7.3</generator>
+ <docs>https://validator.w3.org/feed/docs/rss2.html</docs>
+ <ttl>30</ttl>
+ <image>
+ <url>https://static.pyssg.xyz/images/blog.png</url>
+ <title>pyssg</title>
+ <link>https://pyssg.xyz</link>
+ </image>
+ <item>
+ <title>Test file in subdir</title>
+ <link>https://pyssg.xyz/subdir/test2.html</link>
+ <guid isPermaLink="true">https://pyssg.xyz/subdir/test2.html</guid>
+ <pubDate>Mon, 05 Dec 2022 10:58:41 GMT</pubDate>
+ <description>Second file for testing.</description>
+ <content:encoded><![CDATA[<p>This is a small test for the newly added pyssg.xyz.</p>]]></content:encoded>
+ </item>
+ <item>
+ <title>Index</title>
+ <link>https://pyssg.xyz/test.html</link>
+ <guid isPermaLink="true">https://pyssg.xyz/test.html</guid>
+ <pubDate>Mon, 05 Dec 2022 08:05:10 GMT</pubDate>
+ <description>Index page for pyssg.xyz.</description>
+ <content:encoded><![CDATA[<p>This is a small test for the newly added pyssg.xyz.</p>]]></content:encoded>
+ </item>
+ </channel>
+</rss> \ No newline at end of file
diff --git a/pyssg.xyz/live/pyssg.xyz/sitemap.xml b/pyssg.xyz/live/pyssg.xyz/sitemap.xml
new file mode 100644
index 0000000..63949aa
--- /dev/null
+++ b/pyssg.xyz/live/pyssg.xyz/sitemap.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">
+ <url>
+ <loc>https://pyssg.xyz/subdir/test2.html</loc>
+ <lastmod>2022-12-05</lastmod>
+ <changefreq>weekly</changefreq>
+ <priority>1.0</priority>
+ </url>
+ <url>
+ <loc>https://pyssg.xyz/test.html</loc>
+ <lastmod>2022-12-05</lastmod>
+ <changefreq>weekly</changefreq>
+ <priority>1.0</priority>
+ </url>
+
+</urlset> \ No newline at end of file
diff --git a/pyssg.xyz/dst/subdir/test2.html b/pyssg.xyz/live/pyssg.xyz/subdir/test2.html
index 1f0082f..1f0082f 100644
--- a/pyssg.xyz/dst/subdir/test2.html
+++ b/pyssg.xyz/live/pyssg.xyz/subdir/test2.html
diff --git a/pyssg.xyz/dst/test.html b/pyssg.xyz/live/pyssg.xyz/test.html
index 0ce6abd..0ce6abd 100644
--- a/pyssg.xyz/dst/test.html
+++ b/pyssg.xyz/live/pyssg.xyz/test.html
diff --git a/pyssg.xyz/pyssg_alias.sh b/pyssg.xyz/pyssg_alias.sh
deleted file mode 100755
index 087f6cc..0000000
--- a/pyssg.xyz/pyssg_alias.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-PYSSG_XYZ_DIR=$HOME/pyssg/pyssg.xyz
-
-alias p='pyssg --config $PYSSG_XYZ_DIR/config.yaml'
diff --git a/pyssg.xyz/src/subdir/test2.md b/pyssg.xyz/src/pyssg.xyz/subdir/test2.md
index 1919587..1919587 100644
--- a/pyssg.xyz/src/subdir/test2.md
+++ b/pyssg.xyz/src/pyssg.xyz/subdir/test2.md
diff --git a/pyssg.xyz/src/test.md b/pyssg.xyz/src/pyssg.xyz/test.md
index 8ff6cea..8ff6cea 100644
--- a/pyssg.xyz/src/test.md
+++ b/pyssg.xyz/src/pyssg.xyz/test.md
diff --git a/src/pyssg/builder.py b/src/pyssg/builder.py
index eec0125..65c5837 100644
--- a/src/pyssg/builder.py
+++ b/src/pyssg/builder.py
@@ -27,29 +27,28 @@ class Builder:
if self.dir_path not in self.config['dirs']:
log.error('couldn\'t find "dirs.%s" attribute in config file', self.dir_path)
sys.exit(1)
-
if os.path.isabs(self.dir_path) and self.dir_path.strip() != '/':
log.error('dir path "%s" cannot be absolute, except for the special case "/"', self.dir_path)
sys.exit(1)
- log.debug('building dir_config and src/dst paths for "%s" dir path', self.dir_path)
- self.dir_config: dict = deepcopy(self.config['dirs'][self.dir_path])
+ log.debug('building dir_cfg for "%s" dir_path', self.dir_path)
+ self.dir_cfg: dict = deepcopy(self.config['dirs'][self.dir_path]['cfg'])
if self.dir_path.strip() == '/':
- log.debug('dir path is "/", copying src/dst directly')
- self.dir_config['src'] = self.config['path']['src']
- self.dir_config['dst'] = self.config['path']['dst']
- self.dir_config['url'] = self.config['url']['main']
+ log.debug('dir_path is "/", copying src/dst directly')
+ self.dir_cfg['src'] = self.config['path']['src']
+ self.dir_cfg['dst'] = self.config['path']['dst']
+ self.dir_cfg['url'] = self.config['url']['main']
else:
- self.dir_config['src'] = os.path.join(self.config['path']['src'], self.dir_path)
- self.dir_config['dst'] = os.path.join(self.config['path']['dst'], self.dir_path)
- self.dir_config['url'] = f"{self.config['url']['main']}/{self.dir_path}"
+ log.debug('dir_path is "%s", generating', self.dir_path)
+ self.dir_cfg['src'] = os.path.join(self.config['path']['src'], self.dir_path)
+ self.dir_cfg['dst'] = os.path.join(self.config['path']['dst'], self.dir_path)
+ self.dir_cfg['url'] = f'{self.config["url"]["main"]}/{self.dir_path}'
# the autoescape option could be a security risk if used in a dynamic
# website, as far as i can tell
log.debug('initializing the jinja environment')
- self.__loader: FSLoader = FSLoader(self.config['path']['plt'])
- self.env: Environment = Environment(loader=self.__loader,
+ self.env: Environment = Environment(loader=FSLoader(self.config['path']['plt']),
autoescape=False,
trim_blocks=True,
lstrip_blocks=True)
@@ -64,31 +63,31 @@ class Builder:
self.all_tags: list[tuple[str, str]]
self.common_vars: dict
-
def build(self) -> None:
log.debug('building site for dir path "%s"', self.dir_path)
- if 'exclude_dirs' not in self.dir_config:
- log.debug('"exclude_dirs" attribute not found in "dirs.%s" in config file', self.dir_path)
- self.dir_config['exclude_dirs'] = []
- if not isinstance(self.dir_config['exclude_dirs'], list):
- log.error('"exclude_dirs" attribute is not of type "list"')
+ if 'exclude_dirs' not in self.dir_cfg:
+ log.debug('"exclude_dirs" field not found in "dirs.%s.cfg"', self.dir_path)
+ self.dir_cfg['exclude_dirs'] = []
+ if not isinstance(self.dir_cfg['exclude_dirs'], list):
+ log.error('"exclude_dirs" field in "dirs.%s.cfg" isn\'t of type "list"', self.dir_path)
sys.exit(1)
- self.dirs = get_dir_structure(self.dir_config['src'],
- self.dir_config['exclude_dirs'])
- self.md_files = get_file_list(self.dir_config['src'],
- ['.md'],
- self.dir_config['exclude_dirs'])
- self.html_files = get_file_list(self.dir_config['src'],
- ['.html'],
- self.dir_config['exclude_dirs'])
+ self.dirs = get_dir_structure(self.dir_cfg['src'],
+ self.dir_cfg['exclude_dirs'])
+ self.md_files = get_file_list(self.dir_cfg['src'],
+ tuple('.md'),
+ self.dir_cfg['exclude_dirs'])
+ self.html_files = get_file_list(self.dir_cfg['src'],
+ tuple('.html'),
+ self.dir_cfg['exclude_dirs'])
self.__create_dir_structure()
self.__copy_html_files()
+ # TODO: check if need to pass dirs.dir_path.files
parser: MDParser = MDParser(self.md_files,
self.config,
- self.dir_config,
+ self.dir_cfg,
self.db)
parser.parse_files()
@@ -97,88 +96,89 @@ class Builder:
self.updated_files = parser.updated_files
self.all_tags = parser.all_tags
+ # TODO: check if need to pass dirs.dir_path.files
# dict for the keyword args to pass to the template renderer
- log.debug('adding config, all_pages and all_tags to exposed vars for jinja')
+ log.debug('adding exposed vars for jinja')
self.common_vars = dict(config=self.config,
- dir_config=self.dir_config,
+ dir_config=self.dir_cfg,
all_pages=self.all_files,
all_tags=self.all_tags)
- self.__render_pages(self.dir_config['plt'])
+ self.__render_pages(self.dir_cfg['plt'])
- if 'tags' in self.dir_config and self.dir_config['tags']:
- log.debug('rendering tags for dir "%s"', self.dir_path)
- create_dir(os.path.join(self.dir_config['dst'], 'tag'), True, True)
- self.__render_tags(self.dir_config['tags'])
+ if self.dir_cfg['tags']:
+ log.debug('rendering tags for dir_path "%s"', self.dir_path)
+ create_dir(os.path.join(self.dir_cfg['dst'], 'tag'), True, True)
+ if isinstance(self.dir_cfg['tags'], str):
+ self.__render_tags(self.dir_cfg['tags'])
+ else:
+ self.__render_tags('tag.html')
- opt_renders: dict[str, str] = {'index': 'index.html',
+ default_plts: dict[str, str] = {'index': 'index.html',
'rss': 'rss.xml',
'sitemap': 'sitemap.xml'}
- for opt in opt_renders.keys():
- if opt in self.dir_config and self.dir_config[opt]:
- self.__render_template(self.dir_config[opt],
- opt_renders[opt],
- **self.common_vars)
-
+ for opt in default_plts.keys():
+ if opt in self.dir_cfg:
+ if isinstance(self.dir_cfg[opt], str):
+ self.__render_template(self.dir_cfg[opt],
+ default_plts[opt],
+ **self.common_vars)
+ else:
+ self.__render_template(default_plts[opt],
+ default_plts[opt],
+ **self.common_vars)
def __create_dir_structure(self) -> None:
- log.debug('creating dir structure')
- create_dir(self.dir_config['dst'], True, True)
- _dir_path: str
+ log.debug('creating dir structure for dir_path "%s"', self.dir_path)
+ create_dir(self.dir_cfg['dst'], True, True)
for d in self.dirs:
- _dir_path = os.path.join(self.dir_config['dst'], d)
- # using silent=True to not print the info create dir msgs for this
- create_dir(_dir_path, True, True)
-
+ path: str = os.path.join(self.dir_cfg['dst'], d)
+ create_dir(path, True, True)
def __copy_html_files(self) -> None:
- if len(self.html_files) > 0:
- log.debug('copying all html files')
- else:
+ if not len(self.html_files) > 0:
log.debug('no html files to copy')
+ return
+
+ log.debug('copying all html files')
src_file: str
dst_file: str
-
- for f in self.html_files:
- src_file = os.path.join(self.dir_config['src'], f)
- dst_file = os.path.join(self.dir_config['dst'], f)
-
+ for file in self.html_files:
+ src_file = os.path.join(self.dir_cfg['src'], file)
+ dst_file = os.path.join(self.dir_cfg['dst'], file)
+ # always copy on force
+ if self.config['info']['force']:
+ log.debug('copying "%s"; forced', file)
+ copy_file(src_file, dst_file)
+ continue
# only copy files if they have been modified (or are new)
- if self.db.update(src_file, remove=f'{self.dir_config["src"]}/'):
- log.debug('file "%s" has been modified or is new, copying', f)
+ if self.db.update(src_file, remove=f'{self.dir_cfg["src"]}/'):
+ log.debug('copying "%s"; has been modified or is new', file)
copy_file(src_file, dst_file)
- else:
- if self.config['info']['force']:
- log.debug('file "%s" hasn\'t been modified, but option force is set to true, copying anyways', f)
- copy_file(src_file, dst_file)
- else:
- log.debug('file "%s" hasn\'t been modified, ignoring', f)
-
+ continue
+ log.debug('ignoring "%s"; hasn\'t been modified, not forced', file)
def __render_pages(self, template_name: str) -> None:
- log.debug('rendering html')
+ log.debug('rendering pages with template "%s"', template_name)
page_vars: dict = deepcopy(self.common_vars)
- temp_files: list[Page]
-
+ temp_pages: list[Page]
# check if only updated should be created
if self.config['info']['force']:
log.debug('all html will be rendered, force is set to true')
- temp_files = self.all_files
+ temp_pages = self.all_files
else:
log.debug('only updated or new html will be rendered')
- temp_files = self.updated_files
+ temp_pages = self.updated_files
- for p in temp_files:
- log.debug('adding page to exposed vars for jinja')
+ for p in temp_pages:
+ p_fname: str = p.name.replace('.md', '.html')
+ log.debug('adding page "%s" to exposed vars for jinja', p_fname)
page_vars['page'] = p
# actually render article
- self.__render_template(template_name,
- p.name.replace('.md','.html'),
- **page_vars)
-
+ self.__render_template(template_name, p_fname, **page_vars)
def __render_tags(self, template_name: str) -> None:
- log.debug('rendering tags')
+ log.debug('rendering tags with template "%s"', template_name)
tag_vars: dict = deepcopy(self.common_vars)
tag_pages: list[Page]
for t in self.all_tags:
@@ -192,16 +192,12 @@ class Builder:
log.debug('adding page "%s" as it contains tag "%s"',
p.name, t[0])
tag_pages.append(p)
-
log.debug('adding tag and tag_pages to exposed vars for jinja')
tag_vars['tag'] = t
tag_vars['tag_pages'] = tag_pages
-
+ t_fname: str = f'tag/@{t[0]}.html'
# actually render tag page
- self.__render_template(template_name,
- f'tag/@{t[0]}.html',
- **tag_vars)
-
+ self.__render_template(template_name, t_fname, **tag_vars)
def __render_template(self, template_name: str,
file_name: str,
@@ -210,7 +206,7 @@ class Builder:
file_name, template_name)
template: Template = self.env.get_template(template_name)
content: str = template.render(**template_vars)
- dst_path: str = os.path.join(self.dir_config['dst'], file_name)
+ dst_path: str = os.path.join(self.dir_cfg['dst'], file_name)
log.debug('writing html file to path "%s"', dst_path)
with open(dst_path, 'w') as f:
diff --git a/src/pyssg/configuration.py b/src/pyssg/configuration.py
index 33a82cd..a2b48b6 100644
--- a/src/pyssg/configuration.py
+++ b/src/pyssg/configuration.py
@@ -12,24 +12,37 @@ DEFAULT_CONFIG_PATH: str = '$XDG_CONFIG_HOME/pyssg/config.yaml'
VERSION: str = version('pyssg')
-def __check_well_formed_config(config: dict) -> None:
- log.debug('checking that config file is well formed (at least contains mandatory fields')
- mandatory_config: dict = get_parsed_yaml('mandatory_config.yaml', 'pyssg.plt')[0]
-
- for section in mandatory_config.keys():
- log.debug('checking section "%s"', section)
- if not config[section]:
- log.error('config does not have section "%s"', section)
+def __check_well_formed_config(config: dict,
+ config_base: list[dict],
+ prefix_key: str = '') -> None:
+ for key in config_base[0].keys():
+ current_key: str = f'{prefix_key}.{key}' if prefix_key != '' else key
+ log.debug('checking "%s"', current_key)
+ if key not in config:
+ log.error('config doesn\'t have "%s"', current_key)
+ log.debug('key: %s; config.keys: %s', key, config.keys())
sys.exit(1)
+
+ # checks for dir_paths
+ if key == 'dirs':
+ if '/' not in config[key]:
+ log.error('config doesn\'t have "%s./"', current_key)
+ log.debug('key: %s; config.keys: %s', key, config[key].keys())
+ sys.exit(1)
+
+ log.debug('checking "%s" fields for (%s) dir_paths', key, ', '.join(config[key].keys()))
+ for dkey in config[key].keys():
+ new_current_key: str = f'{current_key}.{dkey}'
+ new_config_base: list[dict] = [config_base[1], config_base[1]]
+ __check_well_formed_config(config[key][dkey], new_config_base, new_current_key)
+ continue
+
# the case for elements that don't have nested elements
- if not mandatory_config[section]:
- log.debug('section "%s" doesn\'t need nested elements', section)
+ if not config_base[0][key]:
+ log.debug('"%s" doesn\'t need nested elements', current_key)
continue
- for option in mandatory_config[section].keys():
- log.debug('checking option "%s"', option)
- if option not in config[section] or not config[section][option]:
- log.error('config does not have option "%s" in section "%s"', option, section)
- sys.exit(1)
+ new_config_base: list[dict] = [config_base[0][key], config_base[1]]
+ __check_well_formed_config(config[key], new_config_base, current_key)
def __expand_all_paths(config: dict) -> None:
@@ -41,11 +54,15 @@ def __expand_all_paths(config: dict) -> None:
# not necessary to type deeper than the first dict
def get_parsed_config(path: str) -> list[dict]:
log.debug('reading config file "%s"', path)
- config: list[dict] = get_parsed_yaml(path) # type: ignore
+ config: list[dict] = get_parsed_yaml(path)
+ mandatory_config: list[dict] = get_parsed_yaml('mandatory_config.yaml', 'pyssg.plt')
log.info('found %s document(s) for configuration "%s"', len(config), path)
-
- __check_well_formed_config(config[0])
+ log.debug('checking that config file is well formed (at least contains mandatory fields')
+ # TODO: make it work with n yaml docs
+ __check_well_formed_config(config[0], mandatory_config)
+ log.error('testing')
+ sys.exit(1)
__expand_all_paths(config[0])
return config
@@ -55,12 +72,12 @@ def get_parsed_config(path: str) -> list[dict]:
# static config means config that shouldn't be changed by the user
def get_static_config() -> dict[str, dict]:
log.debug('reading and setting static config')
- config: dict = get_parsed_yaml('static_config.yaml', 'pyssg.plt')[0] # type: ignore
+ config: dict = get_parsed_yaml('static_config.yaml', 'pyssg.plt')[0]
+ # do I really need a lambda function...
+ current_time = lambda x : datetime.now(tz=timezone.utc).strftime(x)
config['info']['version'] = VERSION
- config['info']['rss_run_date'] = datetime.now(tz=timezone.utc)\
- .strftime(config['fmt']['rss_date'])
- config['info']['sitemap_run_date'] = datetime.now(tz=timezone.utc)\
- .strftime(config['fmt']['sitemap_date'])
+ config['info']['rss_run_date'] = current_time(config['fmt']['rss_date'])
+ config['info']['sitemap_run_date'] = current_time(config['fmt']['sitemap_date'])
return config
diff --git a/src/pyssg/database.py b/src/pyssg/database.py
index 34bf534..d4b6a86 100644
--- a/src/pyssg/database.py
+++ b/src/pyssg/database.py
@@ -19,7 +19,6 @@ class Database:
self.db_path: str = db_path
self.e: dict[str, DatabaseEntry] = dict()
-
# updates the tags for a specific entry (file)
# file_name only contains the entry name (not an absolute path)
def update_tags(self, file_name: str,
@@ -37,7 +36,6 @@ class Database:
' as it is not present in db', file_name)
sys.exit(1)
-
# returns a bool that indicates if the entry
# was (includes new entries) or wasn't updated
def update(self, file_name: str,
@@ -86,7 +84,6 @@ class Database:
log.debug('entry "%s" hasn\'t been modified', f)
return False
-
def write(self) -> None:
log.debug('writing db')
with open(self.db_path, 'w') as file:
@@ -95,7 +92,6 @@ class Database:
csv_writer = csv.writer(file, delimiter=self.__COLUMN_DELIMITER)
csv_writer.writerow(v.get_raw_entry())
-
def _db_path_exists(self) -> bool:
log.debug('checking that "%s" exists or is a file', self.db_path)
if not os.path.exists(self.db_path):
@@ -110,7 +106,6 @@ class Database:
return True
-
def _get_csv_rows(self) -> list[list[str]]:
rows: list[list[str]]
with open(self.db_path, 'r') as f:
@@ -120,7 +115,6 @@ class Database:
return rows
-
def read(self) -> None:
log.debug('reading db')
if not self._db_path_exists():
diff --git a/src/pyssg/plt/default.yaml b/src/pyssg/plt/default.yaml
index 74ef0ee..0b722a6 100644
--- a/src/pyssg/plt/default.yaml
+++ b/src/pyssg/plt/default.yaml
@@ -18,10 +18,11 @@ fmt:
list_sep_date: "%B %Y"
dirs:
/:
- plt: "page.html"
- tags: False
- index: False
- rss: False
- sitemap: False
- exclude_dirs: []
+ cfg:
+ plt: "page.html"
+ tags: False
+ index: False
+ rss: False
+ sitemap: False
+ exclude_dirs: []
... \ No newline at end of file
diff --git a/src/pyssg/plt/mandatory_config.yaml b/src/pyssg/plt/mandatory_config.yaml
index 3f12966..c1ce9f2 100644
--- a/src/pyssg/plt/mandatory_config.yaml
+++ b/src/pyssg/plt/mandatory_config.yaml
@@ -14,5 +14,12 @@ fmt:
list_sep_date:
dirs:
/:
- plt:
+...
+---
+cfg:
+ plt:
+ tags:
+ index:
+ rss:
+ sitemap:
... \ No newline at end of file
diff --git a/src/pyssg/utils.py b/src/pyssg/utils.py
index 3e05d0a..e63ee08 100644
--- a/src/pyssg/utils.py
+++ b/src/pyssg/utils.py
@@ -7,63 +7,65 @@ from logging import Logger, getLogger
log: Logger = getLogger(__name__)
+# TODO: add file exclusion option
def get_file_list(path: str,
- exts: list[str],
- exclude: list[str]=[]) -> list[str]:
+ exts: tuple[str],
+ exclude_dirs: list[str] = []) -> list[str]:
log.debug('retrieving file list in path "%s" that contain file'
- ' extensions (%s) except (%s)',
+ ' extensions (%s) except directories (%s)',
path, ', '.join(exts),
- ', '.join(exclude))
- out: list[str] = []
+ ', '.join(exclude_dirs))
+ file_list: list[str] = []
for root, dirs, files in os.walk(path):
- if exclude != []:
+ if exclude_dirs != []:
log.debug('removing excludes from list')
- dirs[:] = [d for d in dirs if d not in exclude]
-
- for f in files:
- if f.endswith(tuple(exts)):
- stripped_f: str = os.path.join(root, f).replace(path, '')[1:]
- out.append(stripped_f)
+ dirs[:] = [d for d in dirs if d not in exclude_dirs]
+ for file in files:
+ if file.endswith(exts):
+ # [1:] is required to remove the '/' at the beginning after replacing
+ file_name: str = os.path.join(root, file).replace(path, '')[1:]
+ file_list.append(file_name)
log.debug('added file "%s" without "%s" part: "%s"',
- f, path, stripped_f)
+ file, path, file_name)
else:
log.debug('ignoring file "%s" as it doesn\'t contain'
- ' any of the extensions (%s)', f, ', '.join(exts))
-
- return out
+ ' any of the extensions (%s)', file, ', '.join(exts))
+ return file_list
def get_dir_structure(path: str,
- exclude: list[str]=[]) -> list[str]:
- log.debug('retrieving dir structure in path "%s" except (%s)',
+ exclude: list[str] = []) -> list[str]:
+ log.debug('retrieving dir structure in path "%s" except directories (%s)',
path, ', '.join(exclude))
- out: list[str] = []
+ dir_list: list[str] = []
for root, dirs, files in os.walk(path):
if exclude != []:
log.debug('removing excludes from list')
dirs[:] = [d for d in dirs if d not in exclude]
-
for d in dirs:
- if root in out:
- out.remove(root)
+ if root in dir_list:
+ dir_list.remove(root)
log.debug('removed dir "%s" as it already is in the list', root)
+ # not removing the 'path' part here, as comparisons with 'root' would fail
joined_dir: str = os.path.join(root, d)
- out.append(joined_dir)
+ dir_list.append(joined_dir)
log.debug('added dir "%s" to the list', joined_dir)
-
log.debug('removing "%s" from all dirs in list', path)
- return [o.replace(path, '')[1:] for o in out]
+ # [1:] is required to remove the '/' at the beginning after replacing
+ return [d.replace(path, '')[1:] for d in dir_list]
-def create_dir(path: str, p: bool=False, silent=False) -> None:
+def create_dir(path: str, p: bool = False, silent=False) -> None:
try:
if p:
os.makedirs(path)
else:
os.mkdir(path)
- if not silent: log.info('created directory "%s"', path)
+ if not silent:
+ log.info('created directory "%s"', path)
except FileExistsError:
- if not silent: log.info('directory "%s" already exists, ignoring', path)
+ if not silent:
+ log.info('directory "%s" already exists, ignoring', path)
def copy_file(src: str, dst: str) -> None:
@@ -74,6 +76,7 @@ def copy_file(src: str, dst: str) -> None:
log.info('file "%s" already exists, ignoring', dst)
+# only used for database, but keeping it here as it is an independent function
# as seen in SO: https://stackoverflow.com/a/1131238
def get_checksum(path: str) -> str:
log.debug('calculating md5 checksum for "%s"', path)
@@ -81,7 +84,6 @@ def get_checksum(path: str) -> str:
with open(path, "rb") as f:
while chunk := f.read(4096):
file_hash.update(chunk)
-
return file_hash.hexdigest()
@@ -90,8 +92,7 @@ def get_expanded_path(path: str) -> str:
expanded_path: str = os.path.normpath(os.path.expandvars(path))
if '$' in expanded_path:
log.error('"$" character found in expanded path "%s";'
- ' could be due to non-existant env var.', expanded_path)
+ ' could be due to non-existant env var', expanded_path)
sys.exit(1)
log.debug('expanded path "%s" to "%s"', path, expanded_path)
-
return expanded_path
diff --git a/src/pyssg/yaml_parser.py b/src/pyssg/yaml_parser.py
index f9303d6..3109cbc 100644
--- a/src/pyssg/yaml_parser.py
+++ b/src/pyssg/yaml_parser.py
@@ -12,30 +12,23 @@ log: Logger = getLogger(__name__)
def __join_constructor(loader: SafeLoader, node: SequenceNode) -> str:
seq = loader.construct_sequence(node)
return ''.join([str(i) for i in seq])
-SafeLoader.add_constructor('!join', __join_constructor)
-# "file" is either a path or the yaml content itself
-def __read_raw_yaml(file: TextIOWrapper) -> list[dict]:
+def __read_raw_yaml(path: str) -> list[dict]:
all_docs: list[dict] = []
- all_docs_gen = yaml.safe_load_all(file)
- for doc in all_docs_gen:
- all_docs.append(doc)
-
+ with open(path, 'r') as f:
+ for doc in yaml.safe_load_all(f):
+ all_docs.append(doc)
return all_docs
def get_parsed_yaml(resource: str, package: str='') -> list[dict]:
- all_yaml_docs: list[dict] = []
if package == '':
- log.debug('no package specified, reading file "%s"', resource)
- with open(resource, 'r') as f:
- all_yaml_docs = __read_raw_yaml(f)
- else:
- log.debug('package "%s" specified, reading resource "%s"',
- package, resource)
- with rpath(package, resource) as p:
- with open(p, 'r') as f:
- all_yaml_docs = __read_raw_yaml(f)
-
- return all_yaml_docs
+ log.debug('parsing yaml; reading "%s"', resource)
+ return __read_raw_yaml(resource)
+ log.debug('parsing yaml; reading "%s.%s"', package, resource)
+ with rpath(package, resource) as p:
+ return __read_raw_yaml(str(p))
+
+
+SafeLoader.add_constructor('!join', __join_constructor)
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-11-22 13:59:12 +0000