diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index cd098c3..ef27239 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -7,21 +7,27 @@ on: branches: - '**' tags: - - 'v*.*.*' + - 'docs-v*.*.*' release: types: [published] + +defaults: + run: + working-directory: ./docs jobs: build: runs-on: ubuntu-latest - env: - working-directory: ./docs steps: - uses: actions/checkout@v3 - - name: Check documentation build + - name: Install uv for Python 3.13 + uses: astral-sh/setup-uv@v6 + with: + python-version: 3.13 + enable-cache: true + cache-dependency-glob: "docs/uv.lock" + - name: Install the project + run: | + uv sync --all-extras + - name: Build documentation run: | - cd docs - pip3 install -r requirements.txt - make clean - # Fail buiild on any docs warning - # make html O=-W # Removed whilst migrating existing docs - make html \ No newline at end of file + uv run mkdocs build diff --git a/.readthedocs.yml b/.readthedocs.yml index 0b2f93b..8bc102d 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,14 +5,19 @@ version: 2 build: - os: ubuntu-20.04 + os: "ubuntu-24.04" tools: - python: "3.9" + python: "3.13" + jobs: + pre_create_environment: + - cd docs + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + create_environment: + - uv venv "${READTHEDOCS_VIRTUALENV_PATH}" + install: + - cd docs && UV_PROJECT_ENVIRONMENT="${READTHEDOCS_VIRTUALENV_PATH}" uv sync --frozen -sphinx: - configuration: docs/conf.py - fail_on_warning: false # Removed warnings to migrate existing docs - -python: - install: - - requirements: docs/requirements.txt \ No newline at end of file +mkdocs: + configuration: docs/mkdocs.yml \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index b7e8724..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -build: - sphinx-autobuild . _build/ \ No newline at end of file diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..11ec94e --- /dev/null +++ b/docs/README.md @@ -0,0 +1,40 @@ +# CogStack Documentation + +The central documentation for cogstack, hosted on docs.cogstack.org + +## Setup + +### Prerequisites + +- Python 3.10 or higher +- [uv](https://github.com/astral-sh/uv) package manager + +### Installation + +```bash +uv venv --python 3.12 --allow-existing +source .venv/bin/activate +uv sync --dev +``` + +## Usage + +### Serve documentation locally (Development) + +To preview the documentation locally with live reload: + +```bash +uv run mkdocs serve +``` + +The documentation will be available at `http://127.0.0.1:8000` + +### Build documentation + +To build the static site: + +```bash +uv run mkdocs build +``` + +The built site will be in the `site/` directory. \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index 0cf05c1..0000000 --- a/docs/conf.py +++ /dev/null @@ -1,61 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# For the full list of built-in configuration values, see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html -import os -import sys -sys.path.insert(0, os.path.abspath("../observability/docs")) - -print("Hello") -# -- Project information ----------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information - -project = 'CogStack Documentation' -copyright = '2025, CogStack Org' -author = 'CogStack Org' -release = 'latest' -html_title = "CogStack Documentation" - -# -- General configuration --------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration - -extensions = [ - 'sphinx_rtd_theme', - 'sphinx.ext.autodoc', - 'myst_parser', - 'sphinx.ext.inheritance_diagram', - 'sphinx.ext.intersphinx' -] - -templates_path = ['_templates'] -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - - -# -- Options for HTML output ------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output - -html_theme = "furo" -html_static_path = ['_static'] -html_logo = "cogstack-logo.png" - -intersphinx_mapping = { - "sphinx": ("https://www.sphinx-doc.org/en/master/", None), -} -intersphinx_disabled_reftypes = ["*"] - -myst_enable_extensions = [ - "amsmath", - "attrs_inline", - "colon_fence", - "deflist", - "dollarmath", - "fieldlist", - "html_admonition", - "html_image", - # "linkify", - "replacements", - "smartquotes", - "strikethrough", - "substitution", - "tasklist", -] diff --git a/docs/_static/screenshots-dashboards-alerts.png b/docs/docs/_static/screenshots-dashboards-alerts.png similarity index 100% rename from docs/_static/screenshots-dashboards-alerts.png rename to docs/docs/_static/screenshots-dashboards-alerts.png diff --git a/docs/_static/screenshots-dashboards-availability.png b/docs/docs/_static/screenshots-dashboards-availability.png similarity index 100% rename from docs/_static/screenshots-dashboards-availability.png rename to docs/docs/_static/screenshots-dashboards-availability.png diff --git a/docs/_static/screenshots-dashboards-docker-metrics.png b/docs/docs/_static/screenshots-dashboards-docker-metrics.png similarity index 100% rename from docs/_static/screenshots-dashboards-docker-metrics.png rename to docs/docs/_static/screenshots-dashboards-docker-metrics.png diff --git a/docs/_static/screenshots-dashboards-es-metrics.png b/docs/docs/_static/screenshots-dashboards-es-metrics.png similarity index 100% rename from docs/_static/screenshots-dashboards-es-metrics.png rename to docs/docs/_static/screenshots-dashboards-es-metrics.png diff --git a/docs/_static/screenshots-dashboards-vm-metrics.png b/docs/docs/_static/screenshots-dashboards-vm-metrics.png similarity index 100% rename from docs/_static/screenshots-dashboards-vm-metrics.png rename to docs/docs/_static/screenshots-dashboards-vm-metrics.png diff --git a/docs/docs/assets/brand-logo-dark.svg b/docs/docs/assets/brand-logo-dark.svg new file mode 100644 index 0000000..1b28b09 --- /dev/null +++ b/docs/docs/assets/brand-logo-dark.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/docs/assets/brand-logo-dark.svg:Zone.Identifier b/docs/docs/assets/brand-logo-dark.svg:Zone.Identifier new file mode 100644 index 0000000..d6c1ec6 Binary files /dev/null and b/docs/docs/assets/brand-logo-dark.svg:Zone.Identifier differ diff --git a/docs/cogstack-logo.png b/docs/docs/assets/cogstack-logo.png similarity index 100% rename from docs/cogstack-logo.png rename to docs/docs/assets/cogstack-logo.png diff --git a/docs/docs/cogstack-ai/index.md b/docs/docs/cogstack-ai/index.md new file mode 100644 index 0000000..1c139d6 --- /dev/null +++ b/docs/docs/cogstack-ai/index.md @@ -0,0 +1,11 @@ +# CogStack AI + +CogStack AI provides powerful natural language processing tools for healthcare and biomedical text analysis. + +## Available Tools + +- **[MedCAT](https://docs.cogstack.org/projects/nlp)** - Medical Concept Annotation Tool for Named Entity Recognition and Linking (NER+L) from clinical free-text documents. + +- **[MedCAT Trainer](https://docs.cogstack.org/projects/medcat-trainer/)** - Interface for building, improving, and customizing Named Entity Recognition and Linking models for biomedical domain text. + +Click on the links above to access the full documentation for each tool. diff --git a/docs/docs/data-engineering/index.md b/docs/docs/data-engineering/index.md new file mode 100644 index 0000000..f94691b --- /dev/null +++ b/docs/docs/data-engineering/index.md @@ -0,0 +1,9 @@ +# Data Engineering + +CogStack Data Engineering provides tools and frameworks for building scalable, configurable data processing pipelines. + +## Available Tools + +- **[NiFi](https://docs.cogstack.org/projects/nifi/)** - Apache NiFi integration for CogStack, providing fully configurable and scalable data flows with built-in monitoring, data provenance, and security features. + +Click on the link above to access the full documentation. diff --git a/docs/overview/cogstack-documentation.md b/docs/docs/index.md similarity index 65% rename from docs/overview/cogstack-documentation.md rename to docs/docs/index.md index f5539b8..ef9ad81 100644 --- a/docs/overview/cogstack-documentation.md +++ b/docs/docs/index.md @@ -1,21 +1,30 @@ -# CogStack Documentation +Welcome to the CogStack Documentation site. ## What is CogStack? +![CogStack Architecture](overview/attachments/architecture.png) CogStack is a lightweight distributed, fault tolerant database processing architecture and ecosystem, intended to make NLP processing and preprocessing easier in resource constrained environments. It comprises of multiple components, and has been designed to provide configurable data processing pipelines for working with EHR data. CogStack uses databases and files as primary sources of EHR data, with support for custom data connectors. The platform leverages [Apache NiFi](https://nifi.apache.org/) to provide fully configurable data processing pipelines with the goal of generating annotated JSON standardised schema files that can be readily indexed into [ElasticSearch](https://www.elastic.co/), stored as files or pushed back to a database. -![](./attachments/architecture.png) CogStack is a commercial open-source product, with the code available on GitHub: [https://github.com/CogStack/](https://github.com/CogStack/) . For enterprise deployments, full platform setup, and advanced features, please [contact us](https://docs.cogstack.org/en/latest/). -:::{tip} -CogStack is designed as a microservices-based ecosystem. The recommended deployment method is on **Kubernetes using Helm charts**, which provides cloud-native support, scalability, and reliability. Ready-to-use CogStack images are available from the official Docker Hub under the [cogstacksystems](https://hub.docker.com/u/cogstacksystems/) organisation. Docker Compose is still supported for development and smaller deployments, but Kubernetes is recommended for production environments. -::: +!!! tip -## Why does this project exist? + CogStack is designed as a microservices-based ecosystem. The recommended deployment method is on **Kubernetes using Helm charts**, which provides cloud-native support, scalability, and reliability. Ready-to-use CogStack images are available from the official Docker Hub under the [cogstacksystems](https://hub.docker.com/u/cogstacksystems/) organisation. Docker Compose is still supported for development and smaller deployments, but Kubernetes is recommended for production environments. + +## What is CogStack For? CogStack consists of a range of technologies designed to support modern, open source healthcare analytics, and is chiefly comprised of the Elastic stack ([ElasticSearch](https://www.elastic.co/products/elasticsearch), [Kibana](https://www.elastic.co/products/kibana), etc.), [MedCAT](https://github.com/CogStack/MedCAT) (clinical natural language processing for named entity extraction and linking, contextualization, and realtion extraction), clinical text [OCR](https://github.com/CogStack/ocr-service), and clinical text de-identification. Since the processed EHR data can be represented and stored in databases or ElasticSearch, CogStack can be perfectly utilised as one of the solutions for integrating EHR data with other types of biomedical, -omics, wearables data, etc. ---- \ No newline at end of file +--- + +## Community and support + +- **Questions?** Reach out in the [CogStack community forum](https://discourse.cogstack.org/). +- **Code and projects:** [CogStack on GitHub](https://github.com/orgs/CogStack/repositories). + +## Next Steps + +[Get Started ](overview/getting-started.md){ .md-button .md-button--primary } \ No newline at end of file diff --git a/docs/overview/Data pipelines.md b/docs/docs/overview/Data pipelines.md similarity index 77% rename from docs/overview/Data pipelines.md rename to docs/docs/overview/Data pipelines.md index 79f3d5e..01d08ab 100644 --- a/docs/overview/Data pipelines.md +++ b/docs/docs/overview/Data pipelines.md @@ -7,7 +7,19 @@ This page covers the data pipelines used in CogStack ecosystem. ## CogStack-NiFi ### Overview -CogStack-NiFi is the re-architected version of CogStack-Pipeline that replaces the fixed Spring Batch-based pipeline engine with [Apache NiFi](https://nifi.apache.org/). It focuses on fully configurable and scalable data flows with the data processing engine that is easy to use, deploy and tailor to any site-specific data flow requirements. Apache NiFi also comes in with build-in monitoring, data provenance and security features that puts the operations in better control and reliability.
**CogStack-NiFi useful links:**

**Apache NiFi resources:**