From 0b8441c79b047f81526bbb83febc40d7530e35d6 Mon Sep 17 00:00:00 2001 From: David Luevano Alvarado Date: Mon, 5 Dec 2022 05:03:45 -0600 Subject: add extra configuration for more control, add pyssg.xyz example this is the first step towards creating a way to handle multiple subdomains/configs in a single run for a more cohesive site generation --- src/pyssg/builder.py | 100 ++++++++++++++++++++++++++---------- src/pyssg/configuration.py | 2 + src/pyssg/md_parser.py | 9 ++-- src/pyssg/page.py | 39 ++++++++++---- src/pyssg/plt/default.yaml | 11 +++- src/pyssg/plt/mandatory_config.yaml | 4 ++ src/pyssg/pyssg.py | 9 ++-- src/pyssg/yaml_parser.py | 4 -- 8 files changed, 126 insertions(+), 52 deletions(-) (limited to 'src') diff --git a/src/pyssg/builder.py b/src/pyssg/builder.py index 391c7e0..eec0125 100644 --- a/src/pyssg/builder.py +++ b/src/pyssg/builder.py @@ -1,4 +1,6 @@ import os +import sys +import pprint from copy import deepcopy from operator import itemgetter from logging import Logger, getLogger @@ -15,10 +17,33 @@ log: Logger = getLogger(__name__) class Builder: def __init__(self, config: dict, - db: Database): + db: Database, + dir_path: str) -> None: log.debug('initializing site builder') self.config: dict = config self.db: Database = db + self.dir_path: str = dir_path + + if self.dir_path not in self.config['dirs']: + log.error('couldn\'t find "dirs.%s" attribute in config file', self.dir_path) + sys.exit(1) + + if os.path.isabs(self.dir_path) and self.dir_path.strip() != '/': + log.error('dir path "%s" cannot be absolute, except for the special case "/"', self.dir_path) + sys.exit(1) + + log.debug('building dir_config and src/dst paths for "%s" dir path', self.dir_path) + self.dir_config: dict = deepcopy(self.config['dirs'][self.dir_path]) + + if self.dir_path.strip() == '/': + log.debug('dir path is "/", copying src/dst directly') + self.dir_config['src'] = self.config['path']['src'] + self.dir_config['dst'] = self.config['path']['dst'] + self.dir_config['url'] = self.config['url']['main'] + else: + self.dir_config['src'] = os.path.join(self.config['path']['src'], self.dir_path) + self.dir_config['dst'] = os.path.join(self.config['path']['dst'], self.dir_path) + self.dir_config['url'] = f"{self.config['url']['main']}/{self.dir_path}" # the autoescape option could be a security risk if used in a dynamic # website, as far as i can tell @@ -41,21 +66,29 @@ class Builder: def build(self) -> None: - log.debug('building site') - self.dirs = get_dir_structure(self.config['path']['src'], - ['templates']) - self.md_files = get_file_list(self.config['path']['src'], + log.debug('building site for dir path "%s"', self.dir_path) + if 'exclude_dirs' not in self.dir_config: + log.debug('"exclude_dirs" attribute not found in "dirs.%s" in config file', self.dir_path) + self.dir_config['exclude_dirs'] = [] + if not isinstance(self.dir_config['exclude_dirs'], list): + log.error('"exclude_dirs" attribute is not of type "list"') + sys.exit(1) + + self.dirs = get_dir_structure(self.dir_config['src'], + self.dir_config['exclude_dirs']) + self.md_files = get_file_list(self.dir_config['src'], ['.md'], - ['templates']) - self.html_files = get_file_list(self.config['path']['src'], + self.dir_config['exclude_dirs']) + self.html_files = get_file_list(self.dir_config['src'], ['.html'], - ['templates']) + self.dir_config['exclude_dirs']) self.__create_dir_structure() self.__copy_html_files() parser: MDParser = MDParser(self.md_files, self.config, + self.dir_config, self.db) parser.parse_files() @@ -67,23 +100,35 @@ class Builder: # dict for the keyword args to pass to the template renderer log.debug('adding config, all_pages and all_tags to exposed vars for jinja') self.common_vars = dict(config=self.config, + dir_config=self.dir_config, all_pages=self.all_files, all_tags=self.all_tags) - self.__render_articles() - self.__render_tags() - self.__render_template('index.html', 'index.html', **self.common_vars) - self.__render_template('rss.xml', 'rss.xml', **self.common_vars) - self.__render_template('sitemap.xml', 'sitemap.xml', **self.common_vars) + self.__render_pages(self.dir_config['plt']) + + if 'tags' in self.dir_config and self.dir_config['tags']: + log.debug('rendering tags for dir "%s"', self.dir_path) + create_dir(os.path.join(self.dir_config['dst'], 'tag'), True, True) + self.__render_tags(self.dir_config['tags']) + + opt_renders: dict[str, str] = {'index': 'index.html', + 'rss': 'rss.xml', + 'sitemap': 'sitemap.xml'} + for opt in opt_renders.keys(): + if opt in self.dir_config and self.dir_config[opt]: + self.__render_template(self.dir_config[opt], + opt_renders[opt], + **self.common_vars) def __create_dir_structure(self) -> None: log.debug('creating dir structure') - dir_path: str + create_dir(self.dir_config['dst'], True, True) + _dir_path: str for d in self.dirs: - dir_path = os.path.join(self.config['path']['dst'], d) + _dir_path = os.path.join(self.dir_config['dst'], d) # using silent=True to not print the info create dir msgs for this - create_dir(dir_path, True, True) + create_dir(_dir_path, True, True) def __copy_html_files(self) -> None: @@ -95,15 +140,14 @@ class Builder: dst_file: str for f in self.html_files: - src_file = os.path.join(self.config['path']['src'], f) - dst_file = os.path.join(self.config['path']['dst'], f) + src_file = os.path.join(self.dir_config['src'], f) + dst_file = os.path.join(self.dir_config['dst'], f) # only copy files if they have been modified (or are new) - if self.db.update(src_file, remove=f'{self.config["path"]["src"]}/'): + if self.db.update(src_file, remove=f'{self.dir_config["src"]}/'): log.debug('file "%s" has been modified or is new, copying', f) copy_file(src_file, dst_file) else: - # TODO: need to check if this holds after yaml update if self.config['info']['force']: log.debug('file "%s" hasn\'t been modified, but option force is set to true, copying anyways', f) copy_file(src_file, dst_file) @@ -111,9 +155,9 @@ class Builder: log.debug('file "%s" hasn\'t been modified, ignoring', f) - def __render_articles(self) -> None: + def __render_pages(self, template_name: str) -> None: log.debug('rendering html') - article_vars: dict = deepcopy(self.common_vars) + page_vars: dict = deepcopy(self.common_vars) temp_files: list[Page] # check if only updated should be created @@ -126,14 +170,14 @@ class Builder: for p in temp_files: log.debug('adding page to exposed vars for jinja') - article_vars['page'] = p + page_vars['page'] = p # actually render article - self.__render_template("page.html", + self.__render_template(template_name, p.name.replace('.md','.html'), - **article_vars) + **page_vars) - def __render_tags(self) -> None: + def __render_tags(self, template_name: str) -> None: log.debug('rendering tags') tag_vars: dict = deepcopy(self.common_vars) tag_pages: list[Page] @@ -154,7 +198,7 @@ class Builder: tag_vars['tag_pages'] = tag_pages # actually render tag page - self.__render_template('tag.html', + self.__render_template(template_name, f'tag/@{t[0]}.html', **tag_vars) @@ -166,7 +210,7 @@ class Builder: file_name, template_name) template: Template = self.env.get_template(template_name) content: str = template.render(**template_vars) - dst_path: str = os.path.join(self.config['path']['dst'], file_name) + dst_path: str = os.path.join(self.dir_config['dst'], file_name) log.debug('writing html file to path "%s"', dst_path) with open(dst_path, 'w') as f: diff --git a/src/pyssg/configuration.py b/src/pyssg/configuration.py index 1d05289..33a82cd 100644 --- a/src/pyssg/configuration.py +++ b/src/pyssg/configuration.py @@ -43,6 +43,8 @@ def get_parsed_config(path: str) -> list[dict]: log.debug('reading config file "%s"', path) config: list[dict] = get_parsed_yaml(path) # type: ignore + log.info('found %s document(s) for configuration "%s"', len(config), path) + __check_well_formed_config(config[0]) __expand_all_paths(config[0]) diff --git a/src/pyssg/md_parser.py b/src/pyssg/md_parser.py index 5f4fb46..8c61bc5 100644 --- a/src/pyssg/md_parser.py +++ b/src/pyssg/md_parser.py @@ -44,11 +44,13 @@ def _get_md_obj() -> Markdown: class MDParser: def __init__(self, files: list[str], config: dict, + dir_config: dict, db: Database): log.debug('initializing the md parser with %d files', len(files)) self.files: list[str] = files self.config: dict = config + self.dir_config: dict = dir_config self.db: Database = db self.md: Markdown = _get_md_obj() @@ -62,10 +64,10 @@ class MDParser: log.debug('parsing all files') for f in self.files: log.debug('parsing file "%s"', f) - src_file: str = os.path.join(self.config['path']['src'], f) + src_file: str = os.path.join(self.dir_config['src'], f) log.debug('path "%s"', src_file) # get flag if update is successful - file_updated: bool = self.db.update(src_file, remove=f'{self.config["path"]["src"]}/') + file_updated: bool = self.db.update(src_file, remove=f'{self.dir_config["src"]}/') log.debug('parsing md into html') content: str = self.md.reset().convert(open(src_file).read()) @@ -75,7 +77,8 @@ class MDParser: self.db.e[f].mtimestamp, content, self.md.Meta, # type: ignore - self.config) + self.config, + self.dir_config) page.parse_metadata() # keep a separated list for all and updated pages diff --git a/src/pyssg/page.py b/src/pyssg/page.py index 4a12f62..c77e3fa 100644 --- a/src/pyssg/page.py +++ b/src/pyssg/page.py @@ -13,7 +13,8 @@ class Page: mtime: float, html: str, meta: dict, - config: dict): + config: dict, + dir_config: dict) -> None: log.debug('initializing the page object with name "%s"', name) # initial data self.name: str = name @@ -21,7 +22,9 @@ class Page: self.mtimestamp: float = mtime self.content: str = html self.meta: dict = meta + # TODO: need to fix this to use the dir_config stuff self.config: dict = config + self.dir_config: dict = dir_config # data from self.meta self.title: str @@ -66,7 +69,7 @@ class Page: return self.meta[meta][0] except KeyError: log.error('failed to parse mandatory metadata "%s" from file "%s"', - meta, os.path.join(self.config['path']['src'], self.name)) + meta, os.path.join(self.dir_config['src'], self.name)) sys.exit(1) @@ -108,24 +111,38 @@ class Page: tags_only.sort() for t in tags_only: - self.tags.append((t, - f'{self.config["url"]["main"]}/tag/@{t}.html')) + # need to specify dir_config['url'] as it is a hardcoded tag url + self.tags.append((t, f'{self.dir_config["url"]}/tag/@{t}.html')) except KeyError: log.debug('not parsing tags, doesn\'t have any') log.debug('parsing url') + # no need to specify dir_config['url'] as self.name already contains the relative url self.url = f'{self.config["url"]["main"]}/{self.name.replace(".md", ".html")}' log.debug('final url "%s"', self.url) log.debug('parsing image url') - try: - self.image_url = \ - f'{self.config["url"]["static"]}/{self.meta["image_url"][0]}' - except KeyError: + image_url: str + if 'image_url' in self.meta: + image_url = self.meta['image_url'][0] + elif 'default_image' in self.config['url']: log.debug('using default image, no image_url metadata found') - self.image_url = \ - f'{self.config["url"]["static"]}/{self.config["url"]["default_image"]}' - log.debug('final image url "%s"', self.image_url) + image_url = self.config['url']['default_image'] + else: + image_url = '' + + if image_url != '': + if 'static' in self.config['url']: + self.image_url = f'{self.config["url"]["static"]}/{image_url}' + else: + log.debug('no static url set, using main url, this could cause problems') + self.image_url = f'{self.config["url"]["main"]}/{image_url}' + log.debug('final image url "%s"', self.image_url) + else: + self.image_url = '' + log.debug('no image url set for the page, could be because no' + ' "image_url" was found in the metadata and/or no ' + ' "default_image" set in the config file') # if contains open graph elements try: diff --git a/src/pyssg/plt/default.yaml b/src/pyssg/plt/default.yaml index c90d44d..74ef0ee 100644 --- a/src/pyssg/plt/default.yaml +++ b/src/pyssg/plt/default.yaml @@ -7,12 +7,21 @@ path: src: !join [*root, "src"] dst: !join [*root, "dst"] plt: !join [*root, "plt"] + db: !join [*root, ".files"] url: main: "https://example.com" static: "https://static.example.com" - default_image: "/images/default.png" + default_image: "images/default.png" fmt: date: "%a, %b %d, %Y @ %H:%M %Z" list_date: "%b %d" list_sep_date: "%B %Y" +dirs: + /: + plt: "page.html" + tags: False + index: False + rss: False + sitemap: False + exclude_dirs: [] ... \ No newline at end of file diff --git a/src/pyssg/plt/mandatory_config.yaml b/src/pyssg/plt/mandatory_config.yaml index 52bfa04..3f12966 100644 --- a/src/pyssg/plt/mandatory_config.yaml +++ b/src/pyssg/plt/mandatory_config.yaml @@ -5,10 +5,14 @@ path: src: dst: plt: + db: url: main: fmt: date: list_date: list_sep_date: +dirs: + /: + plt: ... \ No newline at end of file diff --git a/src/pyssg/pyssg.py b/src/pyssg/pyssg.py index acf4542..718e043 100644 --- a/src/pyssg/pyssg.py +++ b/src/pyssg/pyssg.py @@ -85,7 +85,8 @@ def main() -> None: if args['init']: log.info('initializing the directory structure and copying over templates') create_dir(config['path']['src']) - create_dir(os.path.join(config['path']['dst'], 'tag'), True) + # dst gets created on builder + # create_dir(config['path']['dst']) create_dir(config['path']['plt']) files: list[str] = ['index.html', 'page.html', @@ -103,12 +104,10 @@ def main() -> None: if args['build']: log.info('building the html files') - # TODO: need to add this to the config and not assume it - db_path: str = os.path.join(config['path']['src'], '.files') - db: Database = Database(db_path) + db: Database = Database(config['path']['db']) db.read() - builder: Builder = Builder(config, db) + builder: Builder = Builder(config, db, "/") builder.build() db.write() diff --git a/src/pyssg/yaml_parser.py b/src/pyssg/yaml_parser.py index 48c2eec..f9303d6 100644 --- a/src/pyssg/yaml_parser.py +++ b/src/pyssg/yaml_parser.py @@ -12,7 +12,6 @@ log: Logger = getLogger(__name__) def __join_constructor(loader: SafeLoader, node: SequenceNode) -> str: seq = loader.construct_sequence(node) return ''.join([str(i) for i in seq]) -log.warning('adding the custom join constructor to yaml.SafeLoader') SafeLoader.add_constructor('!join', __join_constructor) @@ -39,7 +38,4 @@ def get_parsed_yaml(resource: str, package: str='') -> list[dict]: with open(p, 'r') as f: all_yaml_docs = __read_raw_yaml(f) - log.info('found %s document(s) for configuration "%s"', - len(all_yaml_docs), f'{package}.{resource}' if package != '' else resource) - return all_yaml_docs -- cgit v1.2.3-54-g00ecf