From 0b8441c79b047f81526bbb83febc40d7530e35d6 Mon Sep 17 00:00:00 2001 From: David Luevano Alvarado Date: Mon, 5 Dec 2022 05:03:45 -0600 Subject: add extra configuration for more control, add pyssg.xyz example this is the first step towards creating a way to handle multiple subdomains/configs in a single run for a more cohesive site generation --- .gitignore | 4 +- pyssg.xyz/config.yaml | 30 +++++++++++ pyssg.xyz/db.psv | 2 + pyssg.xyz/dst/subdir/test2.html | 19 +++++++ pyssg.xyz/dst/test.html | 19 +++++++ pyssg.xyz/plt/index.html | 32 ++++++++++++ pyssg.xyz/plt/page.html | 25 +++++++++ pyssg.xyz/plt/rss.xml | 39 ++++++++++++++ pyssg.xyz/plt/sitemap.xml | 22 ++++++++ pyssg.xyz/plt/tag.html | 26 ++++++++++ pyssg.xyz/pyssg_alias.sh | 5 ++ pyssg.xyz/src/subdir/test2.md | 6 +++ pyssg.xyz/src/test.md | 6 +++ src/pyssg/builder.py | 100 ++++++++++++++++++++++++++---------- src/pyssg/configuration.py | 2 + src/pyssg/md_parser.py | 9 ++-- src/pyssg/page.py | 39 ++++++++++---- src/pyssg/plt/default.yaml | 11 +++- src/pyssg/plt/mandatory_config.yaml | 4 ++ src/pyssg/pyssg.py | 9 ++-- src/pyssg/yaml_parser.py | 4 -- 21 files changed, 358 insertions(+), 55 deletions(-) create mode 100644 pyssg.xyz/config.yaml create mode 100644 pyssg.xyz/db.psv create mode 100644 pyssg.xyz/dst/subdir/test2.html create mode 100644 pyssg.xyz/dst/test.html create mode 100644 pyssg.xyz/plt/index.html create mode 100644 pyssg.xyz/plt/page.html create mode 100644 pyssg.xyz/plt/rss.xml create mode 100644 pyssg.xyz/plt/sitemap.xml create mode 100644 pyssg.xyz/plt/tag.html create mode 100755 pyssg.xyz/pyssg_alias.sh create mode 100644 pyssg.xyz/src/subdir/test2.md create mode 100644 pyssg.xyz/src/test.md diff --git a/.gitignore b/.gitignore index 8bf6475..035d5c0 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,4 @@ dmypy.json # project specific .vscode/ -site_example/ -dst/**/*.html -.files \ No newline at end of file +site_example/ \ No newline at end of file diff --git a/pyssg.xyz/config.yaml b/pyssg.xyz/config.yaml new file mode 100644 index 0000000..976ab4f --- /dev/null +++ b/pyssg.xyz/config.yaml @@ -0,0 +1,30 @@ +%YAML 1.2 +--- +define: &root_path "$HOME/pyssg/pyssg.xyz/" + +title: "pyssg" +path: + src: !join [*root_path, "src"] + dst: !join [*root_path, "dst"] + plt: !join [*root_path, "plt"] + db: !join [*root_path, "db.psv"] +url: + main: "https://pyssg.xyz" + static: "https://static.pyssg.xyz" + default_image: "images/default.png" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" + list_date: "%b %d" + list_sep_date: "%B %Y" +dirs: + /: + plt: "page.html" + tags: False + index: False + rss: False + sitemap: False + exclude_dirs: [] + articles: + plt: "page.html" + tags: True +... \ No newline at end of file diff --git a/pyssg.xyz/db.psv b/pyssg.xyz/db.psv new file mode 100644 index 0000000..3e140d9 --- /dev/null +++ b/pyssg.xyz/db.psv @@ -0,0 +1,2 @@ +test.md|1670227510.7104242|0.0|00f771e2ad5285488f201809e2b4365e|- +subdir/test2.md|1670237921.0787709|0.0|309bd695d912634400f1d50b65d51ba3|- diff --git a/pyssg.xyz/dst/subdir/test2.html b/pyssg.xyz/dst/subdir/test2.html new file mode 100644 index 0000000..1f0082f --- /dev/null +++ b/pyssg.xyz/dst/subdir/test2.html @@ -0,0 +1,19 @@ + + + + + + Test file in subdir -- pyssg + + +

Test file in subdir

+

By David Luevano

+

Created: Mon, Dec 05, 2022 @ 10:58 UTC

+

Modified:

+ +

This is a small test for the newly added pyssg.xyz.

+ +

Tags: +

+ + diff --git a/pyssg.xyz/dst/test.html b/pyssg.xyz/dst/test.html new file mode 100644 index 0000000..0ce6abd --- /dev/null +++ b/pyssg.xyz/dst/test.html @@ -0,0 +1,19 @@ + + + + + + Index -- pyssg + + +

Index

+

By David Luevano

+

Created: Mon, Dec 05, 2022 @ 08:05 UTC

+

Modified:

+ +

This is a small test for the newly added pyssg.xyz.

+ +

Tags: +

+ + diff --git a/pyssg.xyz/plt/index.html b/pyssg.xyz/plt/index.html new file mode 100644 index 0000000..96d66ef --- /dev/null +++ b/pyssg.xyz/plt/index.html @@ -0,0 +1,32 @@ + + + + + + Index -- {{config['title']}} + + +

Index -- {{config['title']}}

+

Some text here.

+ +

Tags: + {%for t in all_tags%} + {{t[0]}}{{", " if not loop.last else ""}} + {%endfor%} +

+ +

Articles

+ + + diff --git a/pyssg.xyz/plt/page.html b/pyssg.xyz/plt/page.html new file mode 100644 index 0000000..d7f5e43 --- /dev/null +++ b/pyssg.xyz/plt/page.html @@ -0,0 +1,25 @@ + + + + + + {{page.title}} -- {{config['title']}} + + +

{{page.title}}

+

By {{page.author}}

+

Created: {{page.cdate}}

+ {%if page.mdate is not none%} +

Modified: {{page.mdate}}

+ {%endif%} + + {{page.content}} + +

Tags: + {%for t in page.tags%} + {{t[0]}}{{", " if not loop.last else ""}} + {%endfor%} +

+ + + diff --git a/pyssg.xyz/plt/rss.xml b/pyssg.xyz/plt/rss.xml new file mode 100644 index 0000000..6a3eb00 --- /dev/null +++ b/pyssg.xyz/plt/rss.xml @@ -0,0 +1,39 @@ + + + + {{config['title']}} + {{config['url']['main']}} + + Short site description. + en-us + Blog + Copyright 2021 Somebody + some@one.com (Sombody) + some@one.com (Sombody) + {{config['info']['rss_run_date']}} + {{config['info']['rss_run_date']}} + pyssg v{{config['info']['version']}} + https://validator.w3.org/feed/docs/rss2.html + 30 + + {{config['url']['static']}}/images/blog.png + {{config['title']}} + {{config['url']['main']}} + + {%for p in all_pages%} + + {{p.title}} + {{p.url}} + {{p.url}} + {{p.cdate_rss}} + {%for t in p.tags%} + {{t[0].lower().capitalize()}} + {%endfor%} + {{p.summary}} + + + {%endfor%} + + diff --git a/pyssg.xyz/plt/sitemap.xml b/pyssg.xyz/plt/sitemap.xml new file mode 100644 index 0000000..d9ff21b --- /dev/null +++ b/pyssg.xyz/plt/sitemap.xml @@ -0,0 +1,22 @@ + + + {%for p in all_pages%} + + {{p.url}} + {{p.mdate_sitemap if p.mdate_sitemap else p.cdate_sitemap}} + weekly + 1.0 + + {%endfor%} + + {%for t in all_tags%} + + {{t[1]}} + {{config['info']['sitemap_run_date']}} + daily + 0.5 + + {%endfor%} + diff --git a/pyssg.xyz/plt/tag.html b/pyssg.xyz/plt/tag.html new file mode 100644 index 0000000..59cbdf1 --- /dev/null +++ b/pyssg.xyz/plt/tag.html @@ -0,0 +1,26 @@ + + + + + + Posts filtered by {{tag[0]}} -- {{config['title']}} + + +

Posts filtered by {{tag[0]}}

+

Some text here.

+ +

Articles

+ + + diff --git a/pyssg.xyz/pyssg_alias.sh b/pyssg.xyz/pyssg_alias.sh new file mode 100755 index 0000000..087f6cc --- /dev/null +++ b/pyssg.xyz/pyssg_alias.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +PYSSG_XYZ_DIR=$HOME/pyssg/pyssg.xyz + +alias p='pyssg --config $PYSSG_XYZ_DIR/config.yaml' diff --git a/pyssg.xyz/src/subdir/test2.md b/pyssg.xyz/src/subdir/test2.md new file mode 100644 index 0000000..1919587 --- /dev/null +++ b/pyssg.xyz/src/subdir/test2.md @@ -0,0 +1,6 @@ +title: Test file in subdir +author: David Luevano +lang: en +summary: Second file for testing. + +This is a small test for the newly added pyssg.xyz. \ No newline at end of file diff --git a/pyssg.xyz/src/test.md b/pyssg.xyz/src/test.md new file mode 100644 index 0000000..8ff6cea --- /dev/null +++ b/pyssg.xyz/src/test.md @@ -0,0 +1,6 @@ +title: Index +author: David Luevano +lang: en +summary: Index page for pyssg.xyz. + +This is a small test for the newly added pyssg.xyz. \ No newline at end of file diff --git a/src/pyssg/builder.py b/src/pyssg/builder.py index 391c7e0..eec0125 100644 --- a/src/pyssg/builder.py +++ b/src/pyssg/builder.py @@ -1,4 +1,6 @@ import os +import sys +import pprint from copy import deepcopy from operator import itemgetter from logging import Logger, getLogger @@ -15,10 +17,33 @@ log: Logger = getLogger(__name__) class Builder: def __init__(self, config: dict, - db: Database): + db: Database, + dir_path: str) -> None: log.debug('initializing site builder') self.config: dict = config self.db: Database = db + self.dir_path: str = dir_path + + if self.dir_path not in self.config['dirs']: + log.error('couldn\'t find "dirs.%s" attribute in config file', self.dir_path) + sys.exit(1) + + if os.path.isabs(self.dir_path) and self.dir_path.strip() != '/': + log.error('dir path "%s" cannot be absolute, except for the special case "/"', self.dir_path) + sys.exit(1) + + log.debug('building dir_config and src/dst paths for "%s" dir path', self.dir_path) + self.dir_config: dict = deepcopy(self.config['dirs'][self.dir_path]) + + if self.dir_path.strip() == '/': + log.debug('dir path is "/", copying src/dst directly') + self.dir_config['src'] = self.config['path']['src'] + self.dir_config['dst'] = self.config['path']['dst'] + self.dir_config['url'] = self.config['url']['main'] + else: + self.dir_config['src'] = os.path.join(self.config['path']['src'], self.dir_path) + self.dir_config['dst'] = os.path.join(self.config['path']['dst'], self.dir_path) + self.dir_config['url'] = f"{self.config['url']['main']}/{self.dir_path}" # the autoescape option could be a security risk if used in a dynamic # website, as far as i can tell @@ -41,21 +66,29 @@ class Builder: def build(self) -> None: - log.debug('building site') - self.dirs = get_dir_structure(self.config['path']['src'], - ['templates']) - self.md_files = get_file_list(self.config['path']['src'], + log.debug('building site for dir path "%s"', self.dir_path) + if 'exclude_dirs' not in self.dir_config: + log.debug('"exclude_dirs" attribute not found in "dirs.%s" in config file', self.dir_path) + self.dir_config['exclude_dirs'] = [] + if not isinstance(self.dir_config['exclude_dirs'], list): + log.error('"exclude_dirs" attribute is not of type "list"') + sys.exit(1) + + self.dirs = get_dir_structure(self.dir_config['src'], + self.dir_config['exclude_dirs']) + self.md_files = get_file_list(self.dir_config['src'], ['.md'], - ['templates']) - self.html_files = get_file_list(self.config['path']['src'], + self.dir_config['exclude_dirs']) + self.html_files = get_file_list(self.dir_config['src'], ['.html'], - ['templates']) + self.dir_config['exclude_dirs']) self.__create_dir_structure() self.__copy_html_files() parser: MDParser = MDParser(self.md_files, self.config, + self.dir_config, self.db) parser.parse_files() @@ -67,23 +100,35 @@ class Builder: # dict for the keyword args to pass to the template renderer log.debug('adding config, all_pages and all_tags to exposed vars for jinja') self.common_vars = dict(config=self.config, + dir_config=self.dir_config, all_pages=self.all_files, all_tags=self.all_tags) - self.__render_articles() - self.__render_tags() - self.__render_template('index.html', 'index.html', **self.common_vars) - self.__render_template('rss.xml', 'rss.xml', **self.common_vars) - self.__render_template('sitemap.xml', 'sitemap.xml', **self.common_vars) + self.__render_pages(self.dir_config['plt']) + + if 'tags' in self.dir_config and self.dir_config['tags']: + log.debug('rendering tags for dir "%s"', self.dir_path) + create_dir(os.path.join(self.dir_config['dst'], 'tag'), True, True) + self.__render_tags(self.dir_config['tags']) + + opt_renders: dict[str, str] = {'index': 'index.html', + 'rss': 'rss.xml', + 'sitemap': 'sitemap.xml'} + for opt in opt_renders.keys(): + if opt in self.dir_config and self.dir_config[opt]: + self.__render_template(self.dir_config[opt], + opt_renders[opt], + **self.common_vars) def __create_dir_structure(self) -> None: log.debug('creating dir structure') - dir_path: str + create_dir(self.dir_config['dst'], True, True) + _dir_path: str for d in self.dirs: - dir_path = os.path.join(self.config['path']['dst'], d) + _dir_path = os.path.join(self.dir_config['dst'], d) # using silent=True to not print the info create dir msgs for this - create_dir(dir_path, True, True) + create_dir(_dir_path, True, True) def __copy_html_files(self) -> None: @@ -95,15 +140,14 @@ class Builder: dst_file: str for f in self.html_files: - src_file = os.path.join(self.config['path']['src'], f) - dst_file = os.path.join(self.config['path']['dst'], f) + src_file = os.path.join(self.dir_config['src'], f) + dst_file = os.path.join(self.dir_config['dst'], f) # only copy files if they have been modified (or are new) - if self.db.update(src_file, remove=f'{self.config["path"]["src"]}/'): + if self.db.update(src_file, remove=f'{self.dir_config["src"]}/'): log.debug('file "%s" has been modified or is new, copying', f) copy_file(src_file, dst_file) else: - # TODO: need to check if this holds after yaml update if self.config['info']['force']: log.debug('file "%s" hasn\'t been modified, but option force is set to true, copying anyways', f) copy_file(src_file, dst_file) @@ -111,9 +155,9 @@ class Builder: log.debug('file "%s" hasn\'t been modified, ignoring', f) - def __render_articles(self) -> None: + def __render_pages(self, template_name: str) -> None: log.debug('rendering html') - article_vars: dict = deepcopy(self.common_vars) + page_vars: dict = deepcopy(self.common_vars) temp_files: list[Page] # check if only updated should be created @@ -126,14 +170,14 @@ class Builder: for p in temp_files: log.debug('adding page to exposed vars for jinja') - article_vars['page'] = p + page_vars['page'] = p # actually render article - self.__render_template("page.html", + self.__render_template(template_name, p.name.replace('.md','.html'), - **article_vars) + **page_vars) - def __render_tags(self) -> None: + def __render_tags(self, template_name: str) -> None: log.debug('rendering tags') tag_vars: dict = deepcopy(self.common_vars) tag_pages: list[Page] @@ -154,7 +198,7 @@ class Builder: tag_vars['tag_pages'] = tag_pages # actually render tag page - self.__render_template('tag.html', + self.__render_template(template_name, f'tag/@{t[0]}.html', **tag_vars) @@ -166,7 +210,7 @@ class Builder: file_name, template_name) template: Template = self.env.get_template(template_name) content: str = template.render(**template_vars) - dst_path: str = os.path.join(self.config['path']['dst'], file_name) + dst_path: str = os.path.join(self.dir_config['dst'], file_name) log.debug('writing html file to path "%s"', dst_path) with open(dst_path, 'w') as f: diff --git a/src/pyssg/configuration.py b/src/pyssg/configuration.py index 1d05289..33a82cd 100644 --- a/src/pyssg/configuration.py +++ b/src/pyssg/configuration.py @@ -43,6 +43,8 @@ def get_parsed_config(path: str) -> list[dict]: log.debug('reading config file "%s"', path) config: list[dict] = get_parsed_yaml(path) # type: ignore + log.info('found %s document(s) for configuration "%s"', len(config), path) + __check_well_formed_config(config[0]) __expand_all_paths(config[0]) diff --git a/src/pyssg/md_parser.py b/src/pyssg/md_parser.py index 5f4fb46..8c61bc5 100644 --- a/src/pyssg/md_parser.py +++ b/src/pyssg/md_parser.py @@ -44,11 +44,13 @@ def _get_md_obj() -> Markdown: class MDParser: def __init__(self, files: list[str], config: dict, + dir_config: dict, db: Database): log.debug('initializing the md parser with %d files', len(files)) self.files: list[str] = files self.config: dict = config + self.dir_config: dict = dir_config self.db: Database = db self.md: Markdown = _get_md_obj() @@ -62,10 +64,10 @@ class MDParser: log.debug('parsing all files') for f in self.files: log.debug('parsing file "%s"', f) - src_file: str = os.path.join(self.config['path']['src'], f) + src_file: str = os.path.join(self.dir_config['src'], f) log.debug('path "%s"', src_file) # get flag if update is successful - file_updated: bool = self.db.update(src_file, remove=f'{self.config["path"]["src"]}/') + file_updated: bool = self.db.update(src_file, remove=f'{self.dir_config["src"]}/') log.debug('parsing md into html') content: str = self.md.reset().convert(open(src_file).read()) @@ -75,7 +77,8 @@ class MDParser: self.db.e[f].mtimestamp, content, self.md.Meta, # type: ignore - self.config) + self.config, + self.dir_config) page.parse_metadata() # keep a separated list for all and updated pages diff --git a/src/pyssg/page.py b/src/pyssg/page.py index 4a12f62..c77e3fa 100644 --- a/src/pyssg/page.py +++ b/src/pyssg/page.py @@ -13,7 +13,8 @@ class Page: mtime: float, html: str, meta: dict, - config: dict): + config: dict, + dir_config: dict) -> None: log.debug('initializing the page object with name "%s"', name) # initial data self.name: str = name @@ -21,7 +22,9 @@ class Page: self.mtimestamp: float = mtime self.content: str = html self.meta: dict = meta + # TODO: need to fix this to use the dir_config stuff self.config: dict = config + self.dir_config: dict = dir_config # data from self.meta self.title: str @@ -66,7 +69,7 @@ class Page: return self.meta[meta][0] except KeyError: log.error('failed to parse mandatory metadata "%s" from file "%s"', - meta, os.path.join(self.config['path']['src'], self.name)) + meta, os.path.join(self.dir_config['src'], self.name)) sys.exit(1) @@ -108,24 +111,38 @@ class Page: tags_only.sort() for t in tags_only: - self.tags.append((t, - f'{self.config["url"]["main"]}/tag/@{t}.html')) + # need to specify dir_config['url'] as it is a hardcoded tag url + self.tags.append((t, f'{self.dir_config["url"]}/tag/@{t}.html')) except KeyError: log.debug('not parsing tags, doesn\'t have any') log.debug('parsing url') + # no need to specify dir_config['url'] as self.name already contains the relative url self.url = f'{self.config["url"]["main"]}/{self.name.replace(".md", ".html")}' log.debug('final url "%s"', self.url) log.debug('parsing image url') - try: - self.image_url = \ - f'{self.config["url"]["static"]}/{self.meta["image_url"][0]}' - except KeyError: + image_url: str + if 'image_url' in self.meta: + image_url = self.meta['image_url'][0] + elif 'default_image' in self.config['url']: log.debug('using default image, no image_url metadata found') - self.image_url = \ - f'{self.config["url"]["static"]}/{self.config["url"]["default_image"]}' - log.debug('final image url "%s"', self.image_url) + image_url = self.config['url']['default_image'] + else: + image_url = '' + + if image_url != '': + if 'static' in self.config['url']: + self.image_url = f'{self.config["url"]["static"]}/{image_url}' + else: + log.debug('no static url set, using main url, this could cause problems') + self.image_url = f'{self.config["url"]["main"]}/{image_url}' + log.debug('final image url "%s"', self.image_url) + else: + self.image_url = '' + log.debug('no image url set for the page, could be because no' + ' "image_url" was found in the metadata and/or no ' + ' "default_image" set in the config file') # if contains open graph elements try: diff --git a/src/pyssg/plt/default.yaml b/src/pyssg/plt/default.yaml index c90d44d..74ef0ee 100644 --- a/src/pyssg/plt/default.yaml +++ b/src/pyssg/plt/default.yaml @@ -7,12 +7,21 @@ path: src: !join [*root, "src"] dst: !join [*root, "dst"] plt: !join [*root, "plt"] + db: !join [*root, ".files"] url: main: "https://example.com" static: "https://static.example.com" - default_image: "/images/default.png" + default_image: "images/default.png" fmt: date: "%a, %b %d, %Y @ %H:%M %Z" list_date: "%b %d" list_sep_date: "%B %Y" +dirs: + /: + plt: "page.html" + tags: False + index: False + rss: False + sitemap: False + exclude_dirs: [] ... \ No newline at end of file diff --git a/src/pyssg/plt/mandatory_config.yaml b/src/pyssg/plt/mandatory_config.yaml index 52bfa04..3f12966 100644 --- a/src/pyssg/plt/mandatory_config.yaml +++ b/src/pyssg/plt/mandatory_config.yaml @@ -5,10 +5,14 @@ path: src: dst: plt: + db: url: main: fmt: date: list_date: list_sep_date: +dirs: + /: + plt: ... \ No newline at end of file diff --git a/src/pyssg/pyssg.py b/src/pyssg/pyssg.py index acf4542..718e043 100644 --- a/src/pyssg/pyssg.py +++ b/src/pyssg/pyssg.py @@ -85,7 +85,8 @@ def main() -> None: if args['init']: log.info('initializing the directory structure and copying over templates') create_dir(config['path']['src']) - create_dir(os.path.join(config['path']['dst'], 'tag'), True) + # dst gets created on builder + # create_dir(config['path']['dst']) create_dir(config['path']['plt']) files: list[str] = ['index.html', 'page.html', @@ -103,12 +104,10 @@ def main() -> None: if args['build']: log.info('building the html files') - # TODO: need to add this to the config and not assume it - db_path: str = os.path.join(config['path']['src'], '.files') - db: Database = Database(db_path) + db: Database = Database(config['path']['db']) db.read() - builder: Builder = Builder(config, db) + builder: Builder = Builder(config, db, "/") builder.build() db.write() diff --git a/src/pyssg/yaml_parser.py b/src/pyssg/yaml_parser.py index 48c2eec..f9303d6 100644 --- a/src/pyssg/yaml_parser.py +++ b/src/pyssg/yaml_parser.py @@ -12,7 +12,6 @@ log: Logger = getLogger(__name__) def __join_constructor(loader: SafeLoader, node: SequenceNode) -> str: seq = loader.construct_sequence(node) return ''.join([str(i) for i in seq]) -log.warning('adding the custom join constructor to yaml.SafeLoader') SafeLoader.add_constructor('!join', __join_constructor) @@ -39,7 +38,4 @@ def get_parsed_yaml(resource: str, package: str='') -> list[dict]: with open(p, 'r') as f: all_yaml_docs = __read_raw_yaml(f) - log.info('found %s document(s) for configuration "%s"', - len(all_yaml_docs), f'{package}.{resource}' if package != '' else resource) - return all_yaml_docs -- cgit v1.2.3-54-g00ecf