From 1b2d6aff6ccf72fdb292a1f05bb41bf9633a8f55 Mon Sep 17 00:00:00 2001 From: David Luevano Alvarado Date: Tue, 25 Apr 2023 04:03:48 -0600 Subject: refactor tests and add more typing --- README.md | 18 ++-- requirements.txt | 1 + setup.cfg | 1 + src/pyssg/configuration.py | 2 + src/pyssg/md_parser.py | 12 ++- src/pyssg/page.py | 101 +++++++++------------ src/pyssg/plt/default.yaml | 5 - tests/conftest.py | 31 +++---- tests/io_files/__init__.py | 0 tests/io_files/md/__init__.py | 0 tests/io_files/md/a/__init__.py | 0 tests/io_files/md/a/second.md | 12 --- tests/io_files/md/first.md | 10 -- tests/io_files/md/new.md | 10 -- tests/io_files/multiple.yaml | 55 ----------- tests/io_files/multiple_one_doc_error.yaml | 48 ---------- tests/io_files/simple.yaml | 28 ------ tests/io_files/simple_missing_dirs.yaml | 21 ----- tests/io_files/simple_missing_key.yaml | 29 ------ tests/io_files/simple_missing_root_dir.yaml | 22 ----- tests/sample_files/__init__.py | 0 tests/sample_files/checksum.txt | 1 + tests/sample_files/config/__init__.py | 0 tests/sample_files/config/default.yaml | 23 +++++ .../sample_files/config/default_missing_dirs.yaml | 17 ++++ .../config/default_missing_mandatory_key.yaml | 24 +++++ .../config/default_missing_root_dir.yaml | 18 ++++ tests/sample_files/config/multiple_default.yaml | 45 +++++++++ .../config/multiple_default_one_doc_error.yaml | 39 ++++++++ tests/sample_files/md/__init__.py | 0 tests/sample_files/md/a/__init__.py | 0 tests/sample_files/md/a/second.md | 12 +++ tests/sample_files/md/first.md | 10 ++ tests/sample_files/md/new.md | 10 ++ tests/test_configuration.py | 54 +++++------ tests/test_database.py | 9 +- tests/test_utils.py | 23 +++-- tests/test_yaml_parser.py | 12 +-- 38 files changed, 329 insertions(+), 374 deletions(-) delete mode 100644 tests/io_files/__init__.py delete mode 100644 tests/io_files/md/__init__.py delete mode 100644 tests/io_files/md/a/__init__.py delete mode 100644 tests/io_files/md/a/second.md delete mode 100644 tests/io_files/md/first.md delete mode 100644 tests/io_files/md/new.md delete mode 100644 tests/io_files/multiple.yaml delete mode 100644 tests/io_files/multiple_one_doc_error.yaml delete mode 100644 tests/io_files/simple.yaml delete mode 100644 tests/io_files/simple_missing_dirs.yaml delete mode 100644 tests/io_files/simple_missing_key.yaml delete mode 100644 tests/io_files/simple_missing_root_dir.yaml create mode 100644 tests/sample_files/__init__.py create mode 100644 tests/sample_files/checksum.txt create mode 100644 tests/sample_files/config/__init__.py create mode 100644 tests/sample_files/config/default.yaml create mode 100644 tests/sample_files/config/default_missing_dirs.yaml create mode 100644 tests/sample_files/config/default_missing_mandatory_key.yaml create mode 100644 tests/sample_files/config/default_missing_root_dir.yaml create mode 100644 tests/sample_files/config/multiple_default.yaml create mode 100644 tests/sample_files/config/multiple_default_one_doc_error.yaml create mode 100644 tests/sample_files/md/__init__.py create mode 100644 tests/sample_files/md/a/__init__.py create mode 100644 tests/sample_files/md/a/second.md create mode 100644 tests/sample_files/md/first.md create mode 100644 tests/sample_files/md/new.md diff --git a/README.md b/README.md index f971860..4686564 100644 --- a/README.md +++ b/README.md @@ -12,19 +12,15 @@ Initially inspired by Roman Zolotarev's [`ssg5`](https://rgz.ee/bin/ssg5) and [` - [x] Uses [`jinja`](https://jinja.palletsprojects.com/en/3.0.x/) for templating. - [x] Preserves hand-made `*.html` files. - [x] Tag functionality, useful for blog-style sites. - - [ ] Open Graph (and similar) support. - - Technically, this works if you add the correct metadata to the `*.md` files and use the variables available for Jinja. - [x] Build `sitemap.xml` file. - [ ] Include manually added `*.html` files. - [x] Build `rss.xml` file. - - [ ] Join the `static_url` to all relative URLs found to comply with the [RSS 2.0 spec](https://validator.w3.org/feed/docs/rss2.html). - - This would be added to the parsed HTML text extracted from the MD files, so it would be available to the created `*.html` and `*.xml` files. Note that depending on the reader, it will append the URL specified in the RSS file or use the [`xml:base`](https://www.rssboard.org/news/151/relative-links) specified (for example, [newsboat](https://newsboat.org/) parses `xml:base`). - [ ] Include manually added `*.html` files. - [x] YAML for configuration file, uses [`PyYAML`](https://pyyaml.org/). - - [ ] Handle multiple "documents". - - [ ] More complex directory structure to support multiple subdomains and different types of pages. + - [x] Handle multiple "documents". `PyYAML` supports this. + - [x] More complex directory structure to support multiple subdomains and different types of pages. This is supported by using mupltiple "documents" in the `yaml` config file. +- [x] File checksum checking for modification of files. - [ ] Option/change to using an SQL database instead of the custom solution. -- [x] Checksum checking because the timestamp of the file is not enough. - [ ] Use external markdown extensions. ### Markdown features @@ -37,6 +33,7 @@ This program uses the base [`markdown` syntax](https://daringfireball.net/projec - SmartyPants. - Table of Contents. (With defaults as specified [here](https://python-markdown.github.io/extensions/toc/)) - WikiLinks. +- [pymdvar](https://github.com/luevano/pymdvar) (made by me). - [yafg - Yet Another Figure Generator](https://git.sr.ht/~ferruck/yafg) - [Markdown Checklist](https://github.com/FND/markdown-checklist) - [PyMdown Extensions](https://facelessuser.github.io/pymdown-extensions/) @@ -196,19 +193,18 @@ These variables are exposed to use within the templates. The below list is displ - `toc` (`str`): table of contents as taken from `md.toc`. - `toc_tokens` (`list(dict)`): table of contents tokens as taken from `md.toc_tokens`. - `cdatetime` (`datetime.datetime`): creation datetime object of the page. - - `cdate` (`method`): method thtat takes the name of the `fmt.FMT` and applies it to the `cdatetime` object. + - `cdate` (`method`): method that takes the name of the `fmt.FMT` and applies it to the `cdatetime` object. - `cdate_rss` (`str`): formatted `cdatetime` as required by rss. - `cdate_sitemap` (`str`): formatted `cdatetime` as required by sitemap. - `mdatetime` (`datetime.datetime`): modification datetime object of the page. Defaults to `None`. - - `mdate` (`method`): method thtat takes the name of the `fmt.FMT` and applies it to the `mdatetime` object. + - `mdate` (`method`): method that takes the name of the `fmt.FMT` and applies it to the `mdatetime` object. - `mdate_rss` (`str`): formatted `mdatetime` as required by rss. - `mdate_sitemap` (`str`): formatted `mdatetime` as required by sitemap. - `tags` (`list(tuple(str))`): list of tuple of tags of the page, containing the name and the url of the tag, in that order. Defaults to empty list. - `url` (`str`): url of the page, this already includes the `url/main` from config file. - `image_url` (`str`): image url of the page, this already includes the `url/static`. Defaults to the `url/default_image` config option. - `next/previous` (`Page`): reference to the next or previous page object (containing all these attributes). Defaults to `None`. - - `og` (`dict(str, str)`): dict for object graph metadata. - - `meta` (`dict(str, list(str))`): meta dict as obtained from python-markdown, in case you use a meta tag not yet supported, it will be available there. + - `meta` (`dict(str, list(str))`): meta dict as obtained from python-markdown, in case you use a meta tag not directly supported, it will be available there. - `tag` (`tuple(str)`) (`tag.html`): tuple of name and url of the current tag. - `tag_pages` (`list(Page)`) (`tag.html`): similar to `all_pages` but contains all the pages for the current tag. - `all_tags` (`list(tuple(str))`) (all): similar to `page.tags` but contains all the tags. diff --git a/requirements.txt b/requirements.txt index 6078047..807946b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ yafg>=0.3 pymdown-extensions>=9.9.2 pymdvar>=1.0.3 PyYAML>=6.0 +validators>=0.20.0 diff --git a/setup.cfg b/setup.cfg index 1fb6efd..bf05793 100644 --- a/setup.cfg +++ b/setup.cfg @@ -63,6 +63,7 @@ per-file-ignores = arg_parser.py: E501 custom_logger.py: E501 test_database_entry.py: E501 + test_configuration.py: E501 [pbr] skip_authors = True diff --git a/src/pyssg/configuration.py b/src/pyssg/configuration.py index e2dc26b..c7b4248 100644 --- a/src/pyssg/configuration.py +++ b/src/pyssg/configuration.py @@ -12,6 +12,7 @@ DEFAULT_CONFIG_PATH: str = '$XDG_CONFIG_HOME/pyssg/config.yaml' VERSION: str = version('pyssg') +# TODO: add checking for extensions config (such as pymdvar) def __check_well_formed_config(config: dict[str, Any], config_base: list[dict[str, Any]], prefix_key: str = '') -> None: @@ -80,6 +81,7 @@ def get_static_config(sc_package: str = 'static_config.yaml', log.debug('reading and setting static config') config: dict[str, Any] = get_parsed_yaml(sc_package, plt_resource)[0] + # TODO: move this to utils and update the tests def __time(fmt: str) -> str: return datetime.now(tz=timezone.utc).strftime(config['fmt'][fmt]) diff --git a/src/pyssg/md_parser.py b/src/pyssg/md_parser.py index 3ef297d..3b62985 100644 --- a/src/pyssg/md_parser.py +++ b/src/pyssg/md_parser.py @@ -4,6 +4,7 @@ from logging import Logger, getLogger from markdown import Markdown from yafg import YafgExtension +from pymdvar import VariableExtension from markdown_checklist.extension import ChecklistExtension from .database import Database @@ -12,13 +13,17 @@ from .page import Page log: Logger = getLogger(__name__) -def _get_md_obj() -> Markdown: +# TODO: add configuration testing for extensions config (pymdvar for ex) +def get_md_obj(variables: dict[str, str] = dict(), + enable_env: bool = False) -> Markdown: exts: list = ['extra', 'meta', 'sane_lists', 'smarty', 'toc', 'wikilinks', + VariableExtension(variables=variables, + enable_env=enable_env), # stripTitle generates an error when True, # if there is no title attr YafgExtension(stripTitle=False, @@ -50,7 +55,7 @@ class MDParser: self.config: dict = config self.dir_config: dict = dir_config self.db: Database = db - self.md: Markdown = _get_md_obj() + self.md: Markdown = get_md_obj() self.all_files: list[Page] = [] self.all_tags: list[tuple[str, str]] = [] @@ -65,7 +70,8 @@ class MDParser: log.debug('parsing md into html') content: str = self.md.reset().convert(open(src_file).read()) - # ignoring md.Meta type as it is not yet defined (because it is from an extension) + # ignoring md.Meta type as it is not yet defined + # (because it is from an extension) page: Page = Page(f, self.db.e[f].ctimestamp, self.db.e[f].mtimestamp, diff --git a/src/pyssg/page.py b/src/pyssg/page.py index 6b8916d..4902bea 100644 --- a/src/pyssg/page.py +++ b/src/pyssg/page.py @@ -1,22 +1,21 @@ -import sys from datetime import datetime, timezone from logging import Logger, getLogger +from typing import Any log: Logger = getLogger(__name__) class Page: - def __init__(self, - name: str, - ctime: float, - mtime: float, - html: str, - toc: str, - toc_tokens: list[str], - meta: dict, - config: dict, - dir_config: dict) -> None: - log.debug('initializing the page object with name "%s"', name) + def __init__(self, name: str, + ctime: float, + mtime: float, + html: str, + toc: str, + toc_tokens: list[str], + meta: dict[str, Any], + config: dict[str, Any], + dir_config: dict[str, Any]) -> None: + log.debug('initializing a page object with name "%s"', name) # initial data self.name: str = name self.ctimestamp: float = ctime @@ -24,9 +23,9 @@ class Page: self.content: str = html self.toc: str = toc self.toc_tokens: list[str] = toc_tokens - self.meta: dict = meta - self.config: dict = config - self.dir_config: dict = dir_config + self.meta: dict[str, Any] = meta + self.config: dict[str, Any] = config + self.dir_config: dict[str, Any] = dir_config # data from self.meta self.title: str @@ -48,25 +47,25 @@ class Page: self.next: Page | None = None self.previous: Page | None = None - # also from self.meta, but for og metadata - self.og: dict[str, str] = dict() - def __lt__(self, other): return self.ctimestamp < other.ctimestamp - def __get_meta(self, var: str, or_else: str | list[str]) -> str | list[str]: + def __get_meta(self, var: str, + or_else: str | list[str] = '') -> str | list[str] | Any: if var in self.meta: log.debug('getting metadata "%s"', var) return self.meta[var] else: - log.debug('getting metadata "%s" failed, using optional value "%s"', var, or_else) + log.debug('getting metadata "%s" failed, using optional value "%s"', + var, or_else) return or_else def cdate(self, format: str) -> str: if format in self.config['fmt']: return self.cdatetime.strftime(self.config['fmt'][format]) else: - log.warning('format "%s" not found in config["fmt"], returning empty string', format) + log.warning('format "%s" not found in config, returning ' + 'empty string', format) return '' def mdate(self, format: str) -> str: @@ -74,28 +73,32 @@ class Page: log.warning('no mdatetime found, can\'t return a formatted string') return '' if format in self.config['fmt']: - return self.mdatetime.strftime(self.config['fmt'][format]) # type: ignore + return self.mdatetime.strftime(self.config['fmt'][format]) else: - log.warning('format "%s" not found in config["fmt"], returning empty string', format) + log.warning('format "%s" not found in config, returning ' + 'empty string', format) return '' + def from_timestamp(self, timestamp: float) -> datetime: + return datetime.fromtimestamp(timestamp, tz=timezone.utc) + # parses meta from self.meta, for og, it prioritizes, # the actual og meta def parse_metadata(self): log.debug('parsing metadata for file "%s"', self.name) - self.title = self.__get_meta('title', [''])[0] + self.title = str(self.__get_meta('title')) self.author = list(self.__get_meta('author', [''])) - self.summary = self.__get_meta('summary', [''])[0] - self.lang = self.__get_meta('lang', ['en'])[0] + self.summary = str(self.__get_meta('summary')) + self.lang = str(self.__get_meta('lang', 'en')) log.debug('parsing timestamp') - self.cdatetime = datetime.fromtimestamp(self.ctimestamp, tz=timezone.utc) + self.cdatetime = self.from_timestamp(self.ctimestamp) self.cdate_rss = self.cdate('rss_date') self.cdate_sitemap = self.cdate('sitemap_date') if self.mtimestamp != 0.0: log.debug('parsing modified timestamp') - self.mdatetime = datetime.fromtimestamp(self.mtimestamp, tz=timezone.utc) + self.mdatetime = self.from_timestamp(self.mtimestamp) self.mdate_rss = self.mdate('rss_date') self.mdate_sitemap = self.mdate('sitemap_date') else: @@ -108,30 +111,35 @@ class Page: tags_only.sort() for t in tags_only: - # need to specify dir_config['url'] as it is a hardcoded tag url - self.tags.append((t, f'{self.dir_config["url"]}/tag/@{t}.html')) + # need to specify dir_config['url'] as it is + # a hardcoded tag url + tag_url: str = f'{self.dir_config["url"]}/tag/@{t}.html' + self.tags.append((t, tag_url)) else: log.debug('no tags to parse') - log.debug('parsing url') - # no need to specify dir_config['url'] as self.name already contains the relative url - self.url = f'{self.config["url"]["main"]}/{self.name.replace(".md", ".html")}' + log.debug('parsing page url') + # no need to specify dir_config['url'] as self.name already + # contains the relative url + name_html: str = self.name.replace(".md", ".html") + self.url = f'{self.config["url"]["main"]}/{name_html}' log.debug('final url "%s"', self.url) log.debug('parsing image url') default_image_url: str = '' if 'default_image' in self.config['url']: - log.debug('"default_image" url found, will use if no "image_url" is found') + log.debug('"default_image" url found, will use if no "image_url" ' + 'is found') default_image_url = self.config['url']['default_image'] image_url: str - image_url = self.__get_meta('image_url', [default_image_url])[0] + image_url = str(self.__get_meta('image_url', default_image_url)) if image_url != '': if 'static' in self.config['url']: self.image_url = f'{self.config["url"]["static"]}/{image_url}' else: - log.debug('no static url set, using main url, this could cause problems') + log.debug('no static url set, using main url') self.image_url = f'{self.config["url"]["main"]}/{image_url}' log.debug('final image url "%s"', self.image_url) else: @@ -139,24 +147,3 @@ class Page: log.debug('no image url set for the page, could be because no' ' "image_url" was found in the metadata and/or no ' ' "default_image" set in the config file') - - # if contains open graph elements - # TODO: better handle this part - # og_e = object graph entry - og_elements: list[str] = list(self.__get_meta('og', [])) - if og_elements: - log.debug('parsing og metadata') - for og_e in og_elements: - kv: list[str] = og_e.split(',', 1) - if len(kv) != 2: - log.error('invalid og syntax for "%s", needs to be "k, v"', og_e) - sys.exit(1) - - k: str = kv[0].strip() - v: str = kv[1].strip() - - log.debug('og element: ("%s", "%s")', k, v) - self.og[k] = v - - else: - log.debug('no tags to parse') diff --git a/src/pyssg/plt/default.yaml b/src/pyssg/plt/default.yaml index 0b722a6..ca2f7ad 100644 --- a/src/pyssg/plt/default.yaml +++ b/src/pyssg/plt/default.yaml @@ -10,12 +10,8 @@ path: db: !join [*root, ".files"] url: main: "https://example.com" - static: "https://static.example.com" - default_image: "images/default.png" fmt: date: "%a, %b %d, %Y @ %H:%M %Z" - list_date: "%b %d" - list_sep_date: "%B %Y" dirs: /: cfg: @@ -24,5 +20,4 @@ dirs: index: False rss: False sitemap: False - exclude_dirs: [] ... \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index d3f28d7..aaf0b3a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,18 +31,18 @@ def sitemap_date_fmt(): @pytest.fixture(scope='session') -def test_dir() -> str: - return str(os.path.dirname(os.path.abspath(__file__))) +def sample_files_path() -> str: + return f'{str(os.path.dirname(os.path.abspath(__file__)))}/sample_files' @pytest.fixture(scope='session') -def test_resource() -> str: - return 'tests.io_files' +def config_resource() -> str: + return 'tests.sample_files.config' @pytest.fixture(scope='session') -def simple_yaml() -> str: - return 'simple.yaml' +def default_yaml() -> str: + return 'default.yaml' @pytest.fixture(scope='session') @@ -76,7 +76,7 @@ def get_fmt_time() -> Callable[..., str]: @pytest.fixture -def simple_dict() -> dict[str, Any]: +def default_config_dict() -> dict[str, Any]: return {'define': '$PYSSG_HOME/pyssg/site_example/', 'title': 'Example site', 'path': { @@ -85,13 +85,9 @@ def simple_dict() -> dict[str, Any]: 'plt': '/tmp/pyssg/pyssg/site_example/plt', 'db': '/tmp/pyssg/pyssg/site_example/.files'}, 'url': { - 'main': 'https://example.com', - 'static': 'https://static.example.com', - 'default_image': 'images/default.png'}, + 'main': 'https://example.com'}, 'fmt': { - 'date': '%a, %b %d, %Y @ %H:%M %Z', - 'list_date': '%b %d', - 'list_sep_date': '%B %Y'}, + 'date': '%a, %b %d, %Y @ %H:%M %Z'}, 'dirs': { '/': { 'cfg': { @@ -99,13 +95,12 @@ def simple_dict() -> dict[str, Any]: 'tags': False, 'index': False, 'rss': False, - 'sitemap': False, - 'exclude_dirs': []}}}} + 'sitemap': False}}}} @pytest.fixture(scope='function') def tmp_dir_structure(tmp_path: Path) -> Path: - root: Path = tmp_path/'dir_str' + root: Path = tmp_path/'dir_structure' # order matters dirs: list[Path] = [root, root/'first', @@ -165,12 +160,12 @@ def tmp_db_wrong_col_num(tmp_path: Path) -> Path: @pytest.fixture(scope='function') def tmp_src_dir(tmp_path: Path, - test_dir: str) -> Path: + sample_files_path: str) -> Path: src: Path = tmp_path/'src' src_a: Path = src/'a' src.mkdir() src_a.mkdir() - src_test: str = f'{test_dir}/io_files/md' + src_test: str = f'{sample_files_path}/md' files: list[str] = ['first.md', 'new.md', 'a/second.md'] for f in files: diff --git a/tests/io_files/__init__.py b/tests/io_files/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/io_files/md/__init__.py b/tests/io_files/md/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/io_files/md/a/__init__.py b/tests/io_files/md/a/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/io_files/md/a/second.md b/tests/io_files/md/a/second.md deleted file mode 100644 index cb4e333..0000000 --- a/tests/io_files/md/a/second.md +++ /dev/null @@ -1,12 +0,0 @@ -title: Second blog post for testing purposes -author: David Luévano - Someone Else -lang: en -summary: This is just a post used for testing (the second). -tags: test - english - short - update - multiple-author - -A second "blog entry" for testing purposes which uses multiple authors and is inside a subdirectory. diff --git a/tests/io_files/md/first.md b/tests/io_files/md/first.md deleted file mode 100644 index 567ea3e..0000000 --- a/tests/io_files/md/first.md +++ /dev/null @@ -1,10 +0,0 @@ -title: First blog post for testing purposes -author: David Luévano -lang: en -summary: This is just a post used for testing. -tags: test - english - short - update - -Even though I have this "blog" subdomain and page setup, doesn't mean I'll be blogging for pyssg, this is just to serve as an example for the types of sites that pyssg can be used for. \ No newline at end of file diff --git a/tests/io_files/md/new.md b/tests/io_files/md/new.md deleted file mode 100644 index ce684a7..0000000 --- a/tests/io_files/md/new.md +++ /dev/null @@ -1,10 +0,0 @@ -title: New file -author: David Luévano -lang: en -summary: New file used for testing the database. -tags: test - english - short - update - -This is a sample markdown file used for testing pyssg. \ No newline at end of file diff --git a/tests/io_files/multiple.yaml b/tests/io_files/multiple.yaml deleted file mode 100644 index 8d99c40..0000000 --- a/tests/io_files/multiple.yaml +++ /dev/null @@ -1,55 +0,0 @@ -%YAML 1.2 ---- -define: &root "$PYSSG_HOME/pyssg/site_example/" - -title: "Example site" -path: - src: !join [*root, "src"] - dst: !join [*root, "dst"] - plt: !join [*root, "plt"] - db: !join [*root, ".files"] -url: - main: "https://example.com" - static: "https://static.example.com" - default_image: "images/default.png" -fmt: - date: "%a, %b %d, %Y @ %H:%M %Z" - list_date: "%b %d" - list_sep_date: "%B %Y" -dirs: - /: - cfg: - plt: "page.html" - tags: False - index: False - rss: False - sitemap: False - exclude_dirs: [] -... ---- -define: &root "$PYSSG_HOME/pyssg/site_example/" - -title: "Example site" -path: - src: !join [*root, "src"] - dst: !join [*root, "dst"] - plt: !join [*root, "plt"] - db: !join [*root, ".files"] -url: - main: "https://example.com" - static: "https://static.example.com" - default_image: "images/default.png" -fmt: - date: "%a, %b %d, %Y @ %H:%M %Z" - list_date: "%b %d" - list_sep_date: "%B %Y" -dirs: - /: - cfg: - plt: "page.html" - tags: False - index: False - rss: False - sitemap: False - exclude_dirs: [] -... \ No newline at end of file diff --git a/tests/io_files/multiple_one_doc_error.yaml b/tests/io_files/multiple_one_doc_error.yaml deleted file mode 100644 index 86f6546..0000000 --- a/tests/io_files/multiple_one_doc_error.yaml +++ /dev/null @@ -1,48 +0,0 @@ -%YAML 1.2 ---- -define: &root "$PYSSG_HOME/pyssg/site_example/" - -title: "Example site" -path: - src: !join [*root, "src"] - dst: !join [*root, "dst"] - plt: !join [*root, "plt"] - db: !join [*root, ".files"] -url: - main: "https://example.com" - static: "https://static.example.com" - default_image: "images/default.png" -fmt: - date: "%a, %b %d, %Y @ %H:%M %Z" - list_date: "%b %d" - list_sep_date: "%B %Y" -dirs: - /: - cfg: - plt: "page.html" - tags: False - index: False - rss: False - sitemap: False - exclude_dirs: [] -... ---- -define: &root "$PYSSG_HOME/pyssg/site_example/" - -title: "Example site" -path: - src: !join [*root, "src"] - dst: !join [*root, "dst"] - plt: !join [*root, "plt"] - db: !join [*root, ".files"] -url: - main: "https://example.com" - static: "https://static.example.com" - default_image: "images/default.png" -fmt: - date: "%a, %b %d, %Y @ %H:%M %Z" - list_date: "%b %d" - list_sep_date: "%B %Y" -dirs: - # just removing all paths as it will cause an error -... \ No newline at end of file diff --git a/tests/io_files/simple.yaml b/tests/io_files/simple.yaml deleted file mode 100644 index df3888b..0000000 --- a/tests/io_files/simple.yaml +++ /dev/null @@ -1,28 +0,0 @@ -%YAML 1.2 ---- -define: &root "$PYSSG_HOME/pyssg/site_example/" - -title: "Example site" -path: - src: !join [*root, "src"] - dst: !join [*root, "dst"] - plt: !join [*root, "plt"] - db: !join [*root, ".files"] -url: - main: "https://example.com" - static: "https://static.example.com" - default_image: "images/default.png" -fmt: - date: "%a, %b %d, %Y @ %H:%M %Z" - list_date: "%b %d" - list_sep_date: "%B %Y" -dirs: - /: - cfg: - plt: "page.html" - tags: False - index: False - rss: False - sitemap: False - exclude_dirs: [] -... \ No newline at end of file diff --git a/tests/io_files/simple_missing_dirs.yaml b/tests/io_files/simple_missing_dirs.yaml deleted file mode 100644 index aa15fb5..0000000 --- a/tests/io_files/simple_missing_dirs.yaml +++ /dev/null @@ -1,21 +0,0 @@ -%YAML 1.2 ---- -define: &root "$PYSSG_HOME/pyssg/site_example/" - -title: "Example site" -path: - src: !join [*root, "src"] - dst: !join [*root, "dst"] - plt: !join [*root, "plt"] - db: !join [*root, ".files"] -url: - main: "https://example.com" - static: "https://static.example.com" - default_image: "images/default.png" -fmt: - date: "%a, %b %d, %Y @ %H:%M %Z" - list_date: "%b %d" - list_sep_date: "%B %Y" -dirs: -# test missing dirs -... \ No newline at end of file diff --git a/tests/io_files/simple_missing_key.yaml b/tests/io_files/simple_missing_key.yaml deleted file mode 100644 index ac81563..0000000 --- a/tests/io_files/simple_missing_key.yaml +++ /dev/null @@ -1,29 +0,0 @@ -%YAML 1.2 ---- -define: &root "$PYSSG_HOME/pyssg/site_example/" - -# test missing mandatory key -# title: "Example site" -path: - src: !join [*root, "src"] - dst: !join [*root, "dst"] - plt: !join [*root, "plt"] - db: !join [*root, ".files"] -url: - main: "https://example.com" - static: "https://static.example.com" - default_image: "images/default.png" -fmt: - date: "%a, %b %d, %Y @ %H:%M %Z" - list_date: "%b %d" - list_sep_date: "%B %Y" -dirs: - /: - cfg: - plt: "page.html" - tags: False - index: False - rss: False - sitemap: False - exclude_dirs: [] -... \ No newline at end of file diff --git a/tests/io_files/simple_missing_root_dir.yaml b/tests/io_files/simple_missing_root_dir.yaml deleted file mode 100644 index 07fa824..0000000 --- a/tests/io_files/simple_missing_root_dir.yaml +++ /dev/null @@ -1,22 +0,0 @@ -%YAML 1.2 ---- -define: &root "$PYSSG_HOME/pyssg/site_example/" - -title: "Example site" -path: - src: !join [*root, "src"] - dst: !join [*root, "dst"] - plt: !join [*root, "plt"] - db: !join [*root, ".files"] -url: - main: "https://example.com" - static: "https://static.example.com" - default_image: "images/default.png" -fmt: - date: "%a, %b %d, %Y @ %H:%M %Z" - list_date: "%b %d" - list_sep_date: "%B %Y" -dirs: -# test missing /: - something: -... \ No newline at end of file diff --git a/tests/sample_files/__init__.py b/tests/sample_files/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/sample_files/checksum.txt b/tests/sample_files/checksum.txt new file mode 100644 index 0000000..025b879 --- /dev/null +++ b/tests/sample_files/checksum.txt @@ -0,0 +1 @@ +The content of this file is irrelevant as it is only to test the checksum function. \ No newline at end of file diff --git a/tests/sample_files/config/__init__.py b/tests/sample_files/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/sample_files/config/default.yaml b/tests/sample_files/config/default.yaml new file mode 100644 index 0000000..08121a6 --- /dev/null +++ b/tests/sample_files/config/default.yaml @@ -0,0 +1,23 @@ +%YAML 1.2 +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" +dirs: + /: + cfg: + plt: "page.html" + tags: False + index: False + rss: False + sitemap: False +... \ No newline at end of file diff --git a/tests/sample_files/config/default_missing_dirs.yaml b/tests/sample_files/config/default_missing_dirs.yaml new file mode 100644 index 0000000..03ee35a --- /dev/null +++ b/tests/sample_files/config/default_missing_dirs.yaml @@ -0,0 +1,17 @@ +%YAML 1.2 +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" +dirs: +# test missing dirs (doesn't have any) +... \ No newline at end of file diff --git a/tests/sample_files/config/default_missing_mandatory_key.yaml b/tests/sample_files/config/default_missing_mandatory_key.yaml new file mode 100644 index 0000000..b5554f7 --- /dev/null +++ b/tests/sample_files/config/default_missing_mandatory_key.yaml @@ -0,0 +1,24 @@ +%YAML 1.2 +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +# test missing mandatory key +# title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" +dirs: + /: + cfg: + plt: "page.html" + tags: False + index: False + rss: False + sitemap: False +... \ No newline at end of file diff --git a/tests/sample_files/config/default_missing_root_dir.yaml b/tests/sample_files/config/default_missing_root_dir.yaml new file mode 100644 index 0000000..896e141 --- /dev/null +++ b/tests/sample_files/config/default_missing_root_dir.yaml @@ -0,0 +1,18 @@ +%YAML 1.2 +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" +dirs: +# test missing "/" dir in specific + something: +... \ No newline at end of file diff --git a/tests/sample_files/config/multiple_default.yaml b/tests/sample_files/config/multiple_default.yaml new file mode 100644 index 0000000..54954b1 --- /dev/null +++ b/tests/sample_files/config/multiple_default.yaml @@ -0,0 +1,45 @@ +%YAML 1.2 +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" +dirs: + /: + cfg: + plt: "page.html" + tags: False + index: False + rss: False + sitemap: False +... +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" +dirs: + /: + cfg: + plt: "page.html" + tags: False + index: False + rss: False + sitemap: False +... \ No newline at end of file diff --git a/tests/sample_files/config/multiple_default_one_doc_error.yaml b/tests/sample_files/config/multiple_default_one_doc_error.yaml new file mode 100644 index 0000000..44d9beb --- /dev/null +++ b/tests/sample_files/config/multiple_default_one_doc_error.yaml @@ -0,0 +1,39 @@ +%YAML 1.2 +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" +dirs: + /: + cfg: + plt: "page.html" + tags: False + index: False + rss: False + sitemap: False +... +--- +define: &root "$PYSSG_HOME/pyssg/site_example/" + +title: "Example site" +path: + src: !join [*root, "src"] + dst: !join [*root, "dst"] + plt: !join [*root, "plt"] + db: !join [*root, ".files"] +url: + main: "https://example.com" +fmt: + date: "%a, %b %d, %Y @ %H:%M %Z" +dirs: + # just removing all paths as it will cause an error +... \ No newline at end of file diff --git a/tests/sample_files/md/__init__.py b/tests/sample_files/md/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/sample_files/md/a/__init__.py b/tests/sample_files/md/a/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/sample_files/md/a/second.md b/tests/sample_files/md/a/second.md new file mode 100644 index 0000000..cb4e333 --- /dev/null +++ b/tests/sample_files/md/a/second.md @@ -0,0 +1,12 @@ +title: Second blog post for testing purposes +author: David Luévano + Someone Else +lang: en +summary: This is just a post used for testing (the second). +tags: test + english + short + update + multiple-author + +A second "blog entry" for testing purposes which uses multiple authors and is inside a subdirectory. diff --git a/tests/sample_files/md/first.md b/tests/sample_files/md/first.md new file mode 100644 index 0000000..567ea3e --- /dev/null +++ b/tests/sample_files/md/first.md @@ -0,0 +1,10 @@ +title: First blog post for testing purposes +author: David Luévano +lang: en +summary: This is just a post used for testing. +tags: test + english + short + update + +Even though I have this "blog" subdomain and page setup, doesn't mean I'll be blogging for pyssg, this is just to serve as an example for the types of sites that pyssg can be used for. \ No newline at end of file diff --git a/tests/sample_files/md/new.md b/tests/sample_files/md/new.md new file mode 100644 index 0000000..ce684a7 --- /dev/null +++ b/tests/sample_files/md/new.md @@ -0,0 +1,10 @@ +title: New file +author: David Luévano +lang: en +summary: New file used for testing the database. +tags: test + english + short + update + +This is a sample markdown file used for testing pyssg. \ No newline at end of file diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 68f7808..36761eb 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -7,10 +7,10 @@ from pyssg.configuration import get_static_config, get_parsed_config # this test is a bit sketchy, as the way the datetimes are calculated could vary # by milliseconds or even have a difference in seconds -def test_static_default(rss_date_fmt: str, - sitemap_date_fmt: str, - get_fmt_time: Callable[..., str], - version: str) -> None: +def test_static_config(rss_date_fmt: str, + sitemap_date_fmt: str, + get_fmt_time: Callable[..., str], + version: str) -> None: rss_run_date: str = get_fmt_time(rss_date_fmt) sitemap_run_date: str = get_fmt_time(sitemap_date_fmt) sc_dict: dict[str, Any] = {'fmt': {'rss_date': rss_date_fmt, @@ -22,21 +22,21 @@ def test_static_default(rss_date_fmt: str, assert static_config == sc_dict -def test_simple(test_dir: str, - simple_yaml: str, - simple_dict: dict[str, Any]) -> None: - yaml_path: str = f'{test_dir}/io_files/{simple_yaml}' +def test_default_config(sample_files_path: str, + default_yaml: str, + default_config_dict: dict[str, Any]) -> None: + yaml_path: str = f'{sample_files_path}/config/{default_yaml}' yaml: list[dict[str, Any]] = get_parsed_config(yaml_path) assert len(yaml) == 1 - assert yaml[0] == simple_dict + assert yaml[0] == default_config_dict -def test_simple_mising_key(test_dir: str, - caplog: LogCaptureFixture) -> None: +def test_default_config_mising_mandatory_key(sample_files_path: str, + caplog: LogCaptureFixture) -> None: err: tuple[str, int, str] = ('pyssg.configuration', ERROR, 'config doesn\'t have "title"') - yaml_path: str = f'{test_dir}/io_files/simple_missing_key.yaml' + yaml_path: str = f'{sample_files_path}/config/default_missing_mandatory_key.yaml' with pytest.raises(SystemExit) as system_exit: get_parsed_config(yaml_path) assert system_exit.type == SystemExit @@ -44,12 +44,12 @@ def test_simple_mising_key(test_dir: str, assert caplog.record_tuples[-1] == err -def test_simple_mising_dirs(test_dir: str, - caplog: LogCaptureFixture) -> None: +def test_default_config_mising_dirs(sample_files_path: str, + caplog: LogCaptureFixture) -> None: err: tuple[str, int, str] = ('pyssg.configuration', ERROR, 'config doesn\'t have any dirs (dirs.*)') - yaml_path: str = f'{test_dir}/io_files/simple_missing_dirs.yaml' + yaml_path: str = f'{sample_files_path}/config/default_missing_dirs.yaml' with pytest.raises(SystemExit) as system_exit: get_parsed_config(yaml_path) assert system_exit.type == SystemExit @@ -57,12 +57,12 @@ def test_simple_mising_dirs(test_dir: str, assert caplog.record_tuples[-1] == err -def test_simple_root_dir(test_dir: str, - caplog: LogCaptureFixture) -> None: +def test_default_config_root_dir(sample_files_path: str, + caplog: LogCaptureFixture) -> None: err: tuple[str, int, str] = ('pyssg.configuration', ERROR, 'config doesn\'t have "dirs./"') - yaml_path: str = f'{test_dir}/io_files/simple_missing_root_dir.yaml' + yaml_path: str = f'{sample_files_path}/config/default_missing_root_dir.yaml' with pytest.raises(SystemExit) as system_exit: get_parsed_config(yaml_path) assert system_exit.type == SystemExit @@ -71,24 +71,24 @@ def test_simple_root_dir(test_dir: str, # this really just tests that both documents in the yaml file are read, -# multiple.yaml is just simple.yaml with the same document twice, -# shouldn't be an issue as the yaml package handles this -def test_multiple(test_dir: str, simple_dict: dict[str, Any]) -> None: - yaml_path: str = f'{test_dir}/io_files/multiple.yaml' +# both documents are the same (the default.yaml) +def test_multiple_default_config(sample_files_path: str, + default_config_dict: dict[str, Any]) -> None: + yaml_path: str = f'{sample_files_path}/config/multiple_default.yaml' yaml: list[dict[str, Any]] = get_parsed_config(yaml_path) assert len(yaml) == 2 - assert yaml[0] == simple_dict - assert yaml[1] == simple_dict + assert yaml[0] == default_config_dict + assert yaml[1] == default_config_dict # also, this just tests that the checks for a well formed config file are # processed for all documents -def test_multiple_one_doc_error(test_dir: str, - caplog: LogCaptureFixture) -> None: +def test_multiple_default_config_one_doc_error(sample_files_path: str, + caplog: LogCaptureFixture) -> None: err: tuple[str, int, str] = ('pyssg.configuration', ERROR, 'config doesn\'t have any dirs (dirs.*)') - yaml_path: str = f'{test_dir}/io_files/multiple_one_doc_error.yaml' + yaml_path: str = f'{sample_files_path}/config/multiple_default_one_doc_error.yaml' with pytest.raises(SystemExit) as system_exit: get_parsed_config(yaml_path) assert system_exit.type == SystemExit diff --git a/tests/test_database.py b/tests/test_database.py index 7ca597f..c5957e4 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -6,8 +6,9 @@ from pyssg.database import Database from pyssg.database_entry import DatabaseEntry -def test_read_database_no_db(test_dir: str, caplog: LogCaptureFixture) -> None: - path: str = f'{test_dir}/non_existent_db.psv' +def test_read_database_no_db(sample_files_path: str, + caplog: LogCaptureFixture) -> None: + path: str = f'{sample_files_path}/non_existent_db.psv' war: tuple[str, int, str] = ('pyssg.database', WARNING, f'"{path}" doesn\'t exist, will be created ' @@ -18,9 +19,9 @@ def test_read_database_no_db(test_dir: str, caplog: LogCaptureFixture) -> None: assert caplog.record_tuples[-1] == war -def test_read_database_not_a_file(test_dir: str, +def test_read_database_not_a_file(sample_files_path: str, caplog: LogCaptureFixture) -> None: - path: str = test_dir + path: str = sample_files_path err: tuple[str, int, str] = ('pyssg.database', ERROR, f'"{path}" is not a file') diff --git a/tests/test_utils.py b/tests/test_utils.py index 86242c2..75b79c2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -33,13 +33,14 @@ def test_path_expansion_failure(path: str) -> None: assert system_exit.value.code == 1 -def test_checksum(test_dir: str, simple_yaml: str) -> None: - path: str = f'{test_dir}/io_files/{simple_yaml}' - simple_yaml_checksum: str = 'd4f0a3ed56fd530d3ea485dced25534c' +def test_checksum(sample_files_path: str) -> None: + path: str = f'{sample_files_path}/checksum.txt' + simple_yaml_checksum: str = '437b5a0e20d32fc14944c1c00d066303' checksum: str = get_checksum(path) assert checksum == simple_yaml_checksum +# TODO: actually check the existence of the files and not just the log def test_copy_file(tmp_path: Path, caplog: LogCaptureFixture) -> None: src: Path = tmp_path/'src' dst: Path = tmp_path/'dst' @@ -55,7 +56,9 @@ def test_copy_file(tmp_path: Path, caplog: LogCaptureFixture) -> None: assert caplog.record_tuples[-1] == inf -def test_copy_file_failure(tmp_path: Path, caplog: LogCaptureFixture) -> None: +# TODO: actually check the existence of the files and not just the log +def test_copy_file_already_exists(tmp_path: Path, + caplog: LogCaptureFixture) -> None: src: Path = tmp_path/'src' dst: Path = tmp_path/'dst' src.mkdir() @@ -82,7 +85,9 @@ def test_create_dir(tmp_path: Path, caplog: LogCaptureFixture) -> None: assert caplog.record_tuples[-1] == inf -def test_create_dir_failure(tmp_path: Path, caplog: LogCaptureFixture) -> None: +# TODO: actually check the existence of the files and not just the log +def test_create_dir_already_exists(tmp_path: Path, + caplog: LogCaptureFixture) -> None: path: Path = tmp_path/'new_dir' inf: tuple[str, int, str] = ('pyssg.utils', INFO, @@ -106,7 +111,9 @@ def test_create_dirs(tmp_path: Path, caplog: LogCaptureFixture) -> None: assert caplog.record_tuples[-1] == inf -def test_create_dirs_failure(tmp_path: Path, caplog: LogCaptureFixture) -> None: +# TODO: actually check the existence of the files and not just the log +def test_create_dirs_already_exists(tmp_path: Path, + caplog: LogCaptureFixture) -> None: path: Path = tmp_path/'new_dir' sub_path: Path = path/'sub_dir' inf: tuple[str, int, str] = ('pyssg.utils', @@ -131,7 +138,7 @@ def test_dir_structure(tmp_dir_structure: Path, exclude: list[str], exp_dir_str: list[str]) -> None: dir_str: list[str] = get_dir_structure(str(tmp_dir_structure), exclude) - # order doesn't matter + # order doesn't matter, only for checking that both lists contain the same assert sorted(dir_str) == sorted(exp_dir_str) @@ -158,5 +165,5 @@ def test_file_list(tmp_dir_structure: Path, exclude_dirs: list[str], exp_flist: list[str]) -> None: flist: list[str] = get_file_list(str(tmp_dir_structure), exts, exclude_dirs) - # order doesn't matter + # order doesn't matter, only for checking that both lists contain the same assert sorted(flist) == sorted(exp_flist) diff --git a/tests/test_yaml_parser.py b/tests/test_yaml_parser.py index 906c7e6..0d8df96 100644 --- a/tests/test_yaml_parser.py +++ b/tests/test_yaml_parser.py @@ -5,19 +5,19 @@ from pyssg.yaml_parser import get_parsed_yaml # and test the join functionality -def test_yaml_resource_read(simple_yaml: str, test_resource: str) -> None: - yaml: list[dict[str, Any]] = get_parsed_yaml(simple_yaml, test_resource) +def test_yaml_resource_read(default_yaml: str, config_resource: str) -> None: + yaml: list[dict[str, Any]] = get_parsed_yaml(default_yaml, config_resource) assert len(yaml) == 1 -def test_yaml_path_read(test_dir: str) -> None: - yaml_path: str = f'{test_dir}/io_files/simple.yaml' +def test_yaml_path_read(sample_files_path: str, default_yaml: str) -> None: + yaml_path: str = f'{sample_files_path}/config/{default_yaml}' yaml: list[dict[str, Any]] = get_parsed_yaml(yaml_path) assert len(yaml) == 1 -def test_yaml_join(simple_yaml: str, test_resource: str) -> None: - yaml: dict[str, Any] = get_parsed_yaml(simple_yaml, test_resource)[0] +def test_yaml_join(default_yaml: str, config_resource: str) -> None: + yaml: dict[str, Any] = get_parsed_yaml(default_yaml, config_resource)[0] define_str: str = '$PYSSG_HOME/pyssg/site_example/' assert yaml['define'] == define_str assert yaml['path']['src'] == f'{define_str}src' -- cgit v1.2.3-54-g00ecf