diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index fc6bd92e1..72713af73 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,14 +16,12 @@ jobs: - name: Install deps run: | python -m pip install --upgrade pip setuptools - pip install mkdocs - pip install -r mkdocs_plugin/requirements.txt + pip install -r requirements-docs.txt - - name: Install current plugin + - name: Generate test docs run: | - pip install mkdocs_plugin/ - pip freeze + python generate_test_docs.py - name: Build docs run: | - python -m mkdocs build --clean --site-dir html --config-file mkdocs.yml + zensical build -c diff --git a/.readthedocs.yml b/.readthedocs.yml index 8aea41c0b..905a3487b 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -2,10 +2,10 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.7" -mkdocs: - configuration: mkdocs.yml -formats: all -python: - install: - - requirements: mkdocs_plugin/requirements.txt + python: "3" + commands: + - pip install -r requirements-docs.txt + - python generate_test_docs.py + - zensical build -c + - mkdir -p $READTHEDOCS_OUTPUT/html + - cp -r site/* $READTHEDOCS_OUTPUT/html/ diff --git a/docs/index.md b/docs/index.md index 2d91dbb1f..91e01aef4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -90,32 +90,40 @@ More details on test-runner usage in [test-runner docs](test-runner.md) ## Generating documentation [Full docs](https://test-definitions.readthedocs.io) are generated from existing -YAML files. Resulting markdown files are not stored in the repository. In order -to generate documentation locally one needs to follow the steps below: +YAML files. Resulting markdown files are not stored in the repository. -1. create and activate virtualenv +### Using uv + + uv run --with pyyaml --with zensical -- python generate_test_docs.py + uv run --with zensical -- zensical serve + +### Using pip + +1. create and activate virtualenv ``` - virtualenv -p python3 venv + python3 -m venv venv source venv/bin/activate ``` -2. install requirements +2. install requirements + ``` + pip install -r requirements-docs.txt + ``` +3. generate test docs from YAML definitions + ``` + python generate_test_docs.py ``` - pip install -r mkdocs_plugin/requirements.txt +4. run zensical ``` -3. run mkdocs - * local http server - ``` - mkdocs serve - ``` - This will start small http server on http://127.0.0.1:8000 - - * build static docs - ``` - mkdocs build - ``` - This will convert all generated markdown files to HTML files. By default - files are stored in 'site' directory. See [mkdocs documentation](https://www.mkdocs.org/#building-the-site) - for more details. + zensical serve + ``` + +This will start a local http server on http://127.0.0.1:8000. + +To build static HTML instead: + + zensical build + +Files are stored in the 'site' directory. ## Contributing diff --git a/docs/tags.md b/docs/tags.md deleted file mode 100644 index 98e6010a6..000000000 --- a/docs/tags.md +++ /dev/null @@ -1,3 +0,0 @@ -# Tags - - diff --git a/generate_test_docs.py b/generate_test_docs.py new file mode 100644 index 000000000..9be24b161 --- /dev/null +++ b/generate_test_docs.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (C) 2026 Linaro Ltd. +"""Generate markdown documentation from YAML test definitions. + +Walks the test directories, reads YAML files with metadata sections, +and generates markdown pages, an index table, and a tags page. +Run this before building the docs. +""" + +import argparse +import logging +import os + +import yaml + +TABLE_DIRS = ["automated/linux", "automated/android", "manual"] +TABLE_FILENAME = "tests_table" +DOCS_DIR = "docs" + +log = logging.getLogger(__name__) + + +def tag_anchor(tag): + """Convert a tag name to a URL anchor.""" + return tag.lower().replace(" ", "-").replace("/", "") + + +def parse_test_definition(filepath): + """Parse a YAML test definition file. + + Returns a dict with name, description, scope, os, devices, maintainer, + and steps. Returns None if the file has no metadata section. + """ + try: + with open(filepath, "r") as f: + content = yaml.safe_load(f) + except FileNotFoundError: + return None + except yaml.YAMLError as e: + log.warning("%s: invalid YAML: %s", filepath, e) + return None + + if not isinstance(content, dict) or "metadata" not in content: + return None + + metadata = content["metadata"] + if "name" not in metadata: + log.warning("%s: metadata missing 'name'", filepath) + return None + + try: + steps = content["run"]["steps"] + except (KeyError, TypeError): + log.warning("%s: missing run.steps", filepath) + return None + + return { + "name": metadata["name"], + "description": metadata.get("description", ""), + "scope": metadata.get("scope", []), + "os": metadata.get("os", []), + "devices": metadata.get("devices", []), + "maintainer": metadata.get("maintainer", []), + "steps": steps, + } + + +def build_frontmatter(name, scope_list): + """Build YAML frontmatter for a markdown page.""" + lines = ["---", "title: %s" % name] + if scope_list: + lines.append("tags:") + for item in scope_list: + lines.append(" - %s" % item) + lines.append("---") + return "\n".join(lines) + + +def build_md_list(header, items): + """Build a markdown section with a header and bullet list.""" + lines = ["\n## %s\n" % header] + for item in items: + lines.append(" * %s" % item) + return "\n".join(lines) + + +def build_test_page(rel_path, definition): + """Build the full markdown content for a test page.""" + parts = [build_frontmatter(definition["name"], definition["scope"])] + parts.append("\n# %s\n" % rel_path) + + parts.append("\n## Description\n") + parts.append(definition["description"]) + + if definition["maintainer"]: + parts.append(build_md_list("Maintainer", definition["maintainer"])) + else: + parts.append("\n## Maintainer\n") + + parts.append(build_md_list("OS", definition["os"])) + parts.append(build_md_list("Scope", definition["scope"])) + parts.append(build_md_list("Devices", definition["devices"])) + + parts.append("\n## Steps to reproduce\n") + for line in definition["steps"]: + text = str(line) + if text.startswith("#"): + parts.append(" * \\%s" % text) + else: + parts.append(" * %s" % text) + + return "\n".join(parts) + "\n" + + +def write_test_page(filepath, docs_dir, definition): + """Write a markdown page for a single test definition. + + Returns the relative path (without docs_dir prefix) on success. + """ + # strip .yaml extension + rel_path = filepath.rsplit(".", 1)[0] + + out_path = os.path.join(docs_dir, rel_path + ".md") + os.makedirs(os.path.dirname(out_path), exist_ok=True) + + content = build_test_page(rel_path, definition) + with open(out_path, "w") as f: + f.write(content) + + return rel_path + + +def collect_tags(tags, definition, rel_path): + """Add tags from a test definition to the tags dict.""" + for scope in definition["scope"]: + anchor = tag_anchor(scope) + if anchor not in tags: + tags[anchor] = {"label": scope, "pages": []} + tags[anchor]["pages"].append( + {"name": definition["name"], "path": rel_path + ".md"} + ) + + +def collect_table_row(test_tables, table_dirs, definition, rel_path): + """Add a row to the appropriate index table.""" + for table_name in table_dirs: + if rel_path.startswith(table_name): + scope_links = ", ".join( + "[%s](tags.md#%s)" % (s, tag_anchor(s)) for s in definition["scope"] + ) + test_tables[table_name].append( + { + "name": "[%s](%s.md)" % (definition["name"], rel_path), + "description": definition["description"], + "scope": scope_links, + } + ) + break + + +def generate_tags_page(docs_dir, tags): + """Generate the tags index page.""" + path = os.path.join(docs_dir, "tags.md") + lines = ["# Tags\n"] + for anchor in sorted(tags): + entry = tags[anchor] + lines.append('

%s

\n' % (anchor, entry["label"])) + for page in sorted(entry["pages"], key=lambda p: p["name"]): + lines.append("- [%s](%s)\n" % (page["name"], page["path"])) + lines.append("") + with open(path, "w") as f: + f.write("\n".join(lines)) + + +def generate_index(docs_dir, table_dirs, test_tables, table_filename): + """Generate the tests index table.""" + path = os.path.join(docs_dir, table_filename + ".md") + lines = ["# Tests index\n"] + for table_name in table_dirs: + test_table = test_tables[table_name] + lines.append('\n## %s\n' % table_name) + lines.append("| Name | Description | Scope |") + lines.append("| --- | --- | --- |") + for row in sorted(test_table, key=lambda r: r["name"]): + desc = row["description"].replace("\n", "") + lines.append("| %s | %s | %s |" % (row["name"], desc, row["scope"])) + lines.append("") + with open(path, "w") as f: + f.write("\n".join(lines) + "\n") + + +def main(): + parser = argparse.ArgumentParser( + description="Generate markdown docs from YAML test definitions" + ) + parser.add_argument( + "--docs-dir", default=DOCS_DIR, help="Output directory (default: docs)" + ) + parser.add_argument( + "--table-dirs", + nargs="+", + default=TABLE_DIRS, + help="Directories to scan for YAML files", + ) + parser.add_argument( + "--table-file", + default=TABLE_FILENAME, + help="Name of the index table file (without .md)", + ) + parser.add_argument("-v", "--verbose", action="store_true", help="Show warnings") + args = parser.parse_args() + + logging.basicConfig(level=logging.WARNING if args.verbose else logging.ERROR) + + test_tables = {name: [] for name in args.table_dirs} + tags = {} + generated = 0 + + for table_dir in args.table_dirs: + for root, dirs, filenames in os.walk(table_dir): + for filename in filenames: + if not filename.endswith(".yaml"): + continue + filepath = os.path.join(root, filename) + definition = parse_test_definition(filepath) + if definition is None: + continue + rel_path = write_test_page(filepath, args.docs_dir, definition) + collect_tags(tags, definition, rel_path) + collect_table_row(test_tables, args.table_dirs, definition, rel_path) + generated += 1 + + generate_index(args.docs_dir, args.table_dirs, test_tables, args.table_file) + generate_tags_page(args.docs_dir, tags) + print("Generated %d test doc(s) + index + tags (%d tags)" % (generated, len(tags))) + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index 8d4c70d92..000000000 --- a/mkdocs.yml +++ /dev/null @@ -1,18 +0,0 @@ -site_name: Test Definitions -site_url: https://readthedocs.org/test-definitions -repo_url: https://github.com/linaro/test-definitions -copyright: Linaro Ltd. -extra_css: [extra.css] -theme: - name: material -plugins: - - linaro-test-definitions: - table_dirs: - - 'automated/linux' - - 'automated/android' - - 'manual' - - tags: - tags_file: tags.md - - exclude: - glob: - - plans/* diff --git a/mkdocs_plugin/__init__.py b/mkdocs_plugin/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/mkdocs_plugin/pyproject.toml b/mkdocs_plugin/pyproject.toml deleted file mode 100644 index bc8a4c7af..000000000 --- a/mkdocs_plugin/pyproject.toml +++ /dev/null @@ -1,30 +0,0 @@ -[build-system] -requires = [ - "setuptools", - "mkdocs>=1.1", - "tags-macros-plugin@git+https://github.com/mwasilew/mkdocs-plugin-tags.git" -] -build-backend = "setuptools.build_meta" - -[project] -name = "mkdocs-test-definitions-plugin" -version = "1.5" -keywords = [ - "mkdocs", - "python", - "markdown", - "wiki" -] -requires-python = ">=3.5" -license = "GPL-2.0-or-later" - -authors = [ - {name = "Milosz Wasilewski", email = "milosz.wasilewski@oss.qualcomm.com"} -] - -[project.urls] -Repository = "https://github.com/linaro/test-definitions" -GitHub = "https://github.com/linaro/test-definitions" - -[project.entry-points."mkdocs.plugins"] -linaro-test-definitions = "testdefinitionsmkdocs:LinaroTestDefinitionsMkDocsPlugin" diff --git a/mkdocs_plugin/requirements.txt b/mkdocs_plugin/requirements.txt deleted file mode 100644 index 0e7460292..000000000 --- a/mkdocs_plugin/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -mkdocs-test-definitions-plugin -mdutils==1.0.0 -git+https://github.com/mwasilew/mkdocs-plugin-tags.git -mkdocs-exclude -mkdocs-material diff --git a/mkdocs_plugin/testdefinitionsmkdocs/__init__.py b/mkdocs_plugin/testdefinitionsmkdocs/__init__.py deleted file mode 100644 index 862957921..000000000 --- a/mkdocs_plugin/testdefinitionsmkdocs/__init__.py +++ /dev/null @@ -1,155 +0,0 @@ -import errno -import mdutils -import os -import yaml - -from mkdocs.plugins import BasePlugin -from mkdocs.structure.files import File -from mkdocs.config.config_options import Type -from mdutils.fileutils.fileutils import MarkDownFile - - -class LinaroTestDefinitionsMkDocsPlugin(BasePlugin): - config_scheme = ( - ("table_file", Type(str, default="tests_table")), - ("table_dirs", Type(list, default=["automated", "manual"])), - ) - - def __init__(self): - self.table_dirs = ["automated", "manual"] - self.table_filename = "tests_table" - self.test_tables = {} - - def on_config(self, config, **kwargs): - self.table_filename = self.config.get("table_file", self.table_filename) - self.table_dirs = self.config.get("table_dirs", self.table_dirs) - for name in self.table_dirs: - self.test_tables[name] = [] - - def __add_list_with_header(self, mdFile, header_string, item_list): - mdFile.new_header(level=2, title=header_string) - if item_list is not None: - for item in item_list: - mdFile.new_line(" * %s" % item) - - def generate_yaml_markdown(self, filename, config): - # remove leading ./ - new_filename = filename.split("/", 1)[1] - # remove .yaml - new_filename = new_filename.rsplit(".", 1)[0] - tmp_filename = os.path.join(config["docs_dir"], new_filename) - filecontent = None - try: - with open(filename, "r") as f: - filecontent = f.read() - except FileNotFoundError: - return None - try: - content = yaml.load(filecontent, Loader=yaml.Loader) - if "metadata" in content.keys(): - metadata = content["metadata"] - mdFile = mdutils.MdUtils(file_name=tmp_filename) - tags_section = "---\n" - tags_section += "title: %s\n" % metadata["name"] - scope_list = metadata.get("scope", []) - os_list = metadata.get("os", []) - device_list = metadata.get("devices", []) - if scope_list: - tags_section += "tags:\n" - for item in scope_list: - tags_section += " - %s\n" % item - tags_section += "---\n" - mdFile.new_header(level=1, title=new_filename) - mdFile.new_header(level=2, title="Description") - mdFile.write(metadata["description"]) - mdFile.new_header(level=2, title="Maintainer") - maintainer_list = metadata.get("maintainer", None) - if maintainer_list is not None: - for item in maintainer_list: - mdFile.new_line(" * %s" % item) - self.__add_list_with_header(mdFile, "OS", os_list) - self.__add_list_with_header(mdFile, "Scope", scope_list) - self.__add_list_with_header(mdFile, "Devices", device_list) - mdFile.new_header(level=2, title="Steps to reproduce") - steps_list = content["run"]["steps"] - for line in steps_list: - bullet_string = " * " - if str(line).startswith("#"): - bullet_string = " * \\" - mdFile.new_line(bullet_string + str(line)) - try: - os.makedirs(os.path.dirname(tmp_filename)) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - md_file = MarkDownFile(mdFile.file_name) - md_file.rewrite_all_file( - data=tags_section - + mdFile.title - + mdFile.table_of_contents - + mdFile.file_data_text - ) - # add row to tests_table - table_key = None - for table_name in self.table_dirs: - if new_filename.startswith(table_name): - table_key = table_name - if table_key is not None: - self.test_tables[table_key].append( - { - "name": "[%s](%s.md)" % (metadata["name"], new_filename), - "description": metadata["description"], - "scope": ", ".join( - [ - "[%s](tags.md#%s)" - % (x, x.lower().replace(" ", "-").replace("/", "")) - for x in scope_list - ] - ), - } - ) - return new_filename + ".md" - except yaml.YAMLError: - return None - except KeyError: - return None - - def on_files(self, files, config): - for root, dirs, filenames in os.walk("."): - for filename in filenames: - if filename.endswith(".yaml"): - new_filename = os.path.join(root, filename) - markdown_filename = self.generate_yaml_markdown( - new_filename, config - ) - if markdown_filename is not None: - f = File( - markdown_filename, - config["docs_dir"], - config["site_dir"], - False, - ) - files.append(f) - mdFile = mdutils.MdUtils( - file_name=config["docs_dir"] + "/" + self.table_filename - ) - mdFile.new_header(level=1, title="Tests index") - for table_name, test_table in self.test_tables.items(): - mdFile.new_header(level=2, title='%s' % table_name) - mdFile.new_line("| Name | Description | Scope |") - mdFile.new_line("| --- | --- | --- |") - for row in sorted(test_table, key=lambda item: item["name"]): - mdFile.new_line( - "| %s | %s | %s |" - % (row["name"], row["description"].replace("\n", ""), row["scope"]) - ) - mdFile.new_line("") - mdFile.create_md_file() - newfile = File( - path=str(self.table_filename) + ".md", - src_dir=config["docs_dir"], - dest_dir=config["site_dir"], - use_directory_urls=False, - ) - files.append(newfile) - return sorted(files, key=lambda x: x.src_path, reverse=False) diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 000000000..a45425b95 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,2 @@ +pyyaml +zensical diff --git a/zensical.toml b/zensical.toml new file mode 100644 index 000000000..506cb2dae --- /dev/null +++ b/zensical.toml @@ -0,0 +1,9 @@ +[project] +site_name = "Test Definitions" +site_url = "https://readthedocs.org/test-definitions" +repo_url = "https://github.com/linaro/test-definitions" +copyright = "Linaro Ltd." +extra_css = ["extra.css"] + +[project.theme] +variant = "classic"