diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 00000000..72cc2ccf --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,53 @@ +name: Documentation + +on: + push: + pull_request: + branches: + - '**' + +permissions: {} + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: '3.14' + cache: 'pip' + - name: Install documentation dependencies + run: make docs-install + - name: Build documentation + run: make docs-build + - name: Upload Pages artifact + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 + with: + path: site + + deploy: + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + needs: build + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 diff --git a/.gitignore b/.gitignore index 51c08296..820ee873 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ build cover dist docs/_build +site/ lib/PyLD.egg-info profiler tests/test_caching.py diff --git a/AGENTS.md b/AGENTS.md index df89c8d6..74a52f1c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # Agent guidelines -Read [CONTRIBUTING.rst](CONTRIBUTING.rst) for code style, linting (e.g. `make lint`, `make fmt`), and release process. +Read [CONTRIBUTING.md](CONTRIBUTING.md) for code style, linting (e.g. `make lint`, `make fmt`), and release process. ## Testing @@ -10,7 +10,14 @@ Read [CONTRIBUTING.rst](CONTRIBUTING.rst) for code style, linting (e.g. `make li ## Documentation -- When adding or promoting public top-level API exports, reflect them in the project documentation, especially the Sphinx API reference under `docs/`. +- Put docs-specific CSS in `docs/stylesheets/extra.css` and register it via `extra_css` in `mkdocs.yml`. +- Put runnable doc examples in `docs/examples/` and embed them with the `example()` macro in `docs_macros.py`. +- One page = one idea — do not combine unrelated topics on a single doc page. +- Python object names in docs must always use backticks — in prose, headings, and card link text (not `__bold__`). +- Doc examples must not use `set_document_loader()`; pass `documentLoader` per operation via `options`. +- Do not use or mention function-based document loaders (`requests_document_loader`, `aiohttp_document_loader`, or plain callables) in docs; use `*DocumentLoader` classes or a `DocumentLoader` subclass. +- Doc page H1 headers should use icons that match their card icons (e.g. `# :material-cloud-download: \`RequestsDocumentLoader\``). +- Custom document-loader docs should illustrate subclassing `DocumentLoader`, not a bare callable. ## Committing diff --git a/CHANGELOG.md b/CHANGELOG.md index a8205cca..36b1c989 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,9 @@ - The `pytest` test runner now uses plain assert result == expect instead of printing EXPECTED / ACTUAL and raising a generic failure. This enables `pytests`'s native result comparison. +- Convert `./README.rst` and `./CONTRIBUTING.rst` (reStructuredText) to + `./README.md` and `./CONTRIBUTING.md` (markdown). Also update their + contents to reflect the current state of the repo. ### Added - `pyld.DocumentLoader` abstract base class for class-based document loaders, @@ -287,7 +290,7 @@ - **1.0.0**! - [Semantic Versioning](https://semver.org/) is now past the "initial development" 0.x.y stage (after 6+ years!). -- [Conformance](README.rst#conformance): +- [Conformance](README.md#conformance): - JSON-LD 1.0 + JSON-LD 1.0 errata - JSON-LD 1.1 drafts - Thanks to the JSON-LD and related communities and the many many people over diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..fc565910 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,80 @@ +# Contributing to PyLD + +Want to contribute to PyLD? Great! Here are a few notes: + +## Code + +* In general, follow the common [PEP 8 Style Guide](https://www.python.org/dev/peps/pep-0008/). +* Try to make the code pass [ruff](https://docs.astral.sh/ruff/) checks. + + * `make lint` or `ruff check lib/pyld/*` + * You can also apply automatic fixing and formatting + using `make fmt` + +* Use version `X.Y.Z-dev` in dev mode. +* Use version `X.Y.Z` for releases. + +## Documentation + +The public documentation site is built with MkDocs Material. + +* Install documentation dependencies: + + * `make docs-install` + +* Build the site: + + * `make docs-build` + +* Preview documentation locally: + + * `make docs-serve` (override port with `PORT=8008 make docs-serve`) + +* Refresh bundled JSON-LD context files: + + * `make download-bundled-contexts` + +## Versioning + +* Follow the [Semantic Versioning](https://semver.org/) guidelines. + +## Release Process + +* `$EDITOR CHANGELOG.md`: update CHANGELOG with new notes, version, and date. +* commit changes +* `$EDITOR lib/pyld/__about__.py`: update to release version and remove `-dev` suffix. +* `git commit CHANGELOG.md lib/pyld/__about__.py -m "Release {version}."` +* `git tag {version}` +* `$EDITOR lib/pyld/__about__.py`: update to next version and add `-dev` suffix. +* `git commit lib/pyld/__about__.py -m "Start {next-version}."` +* `git push --tags` + +To ensure a clean [package](https://pypi.org/project/PyLD/) upload to [PyPI](https://pypi.org/), +use a clean checkout, and run the following: + +* For more info, look at the packaging + [guide](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/). +* Setup an [API token](https://pypi.org/help/#apitoken). Recommend using a + specific "PyLD" token and set it up as a "repository" in your + [`~/.pypirc`](https://packaging.python.org/en/latest/specifications/pypirc/) + for use in the upload command. +* The below builds and uploads a sdist and wheel. Adjust as needed depending + on how you manage and clean "dist/" dir files. +* `git checkout {version}` +* `python3 -m build` +* `twine check dist/*` +* `twine upload -r PyLD dist/*` + +## Implementation Report Process + +As of early 2020, the process to generate an EARL report for the official +[JSON-LD Processor Conformance](https://w3c.github.io/json-ld-api/reports/) page is: + +* Run the tests on the `json-ld-api` and `json-ld-framing` test repos to + generate a `.jsonld` test report as explained in [README.md](./README.md#tests) +* Use the [rdf](https://rubygems.org/gems/rdf) tool to generate a `.ttl`: + + * `rdf serialize pyld-earl.jsonld --output-format turtle -o pyld-earl.ttl` + +* Optionally follow the [report instructions](https://github.com/w3c/json-ld-api/tree/master/reports) to generate the HTML report for inspection. +* Submit a PR to the [json-ld-api repository](https://github.com/w3c/json-ld-api/pulls) with at least the `.ttl`. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst deleted file mode 100644 index 5053e549..00000000 --- a/CONTRIBUTING.rst +++ /dev/null @@ -1,76 +0,0 @@ -Contributing to PyLD -==================== - -Want to contribute to PyLD? Great! Here are a few notes: - -Code ----- - -* In general, follow the common `PEP 8 Style Guide`_. -* Try to make the code pass ruff_ checks. - - * ``make lint`` or ``ruff check lib/pyld/*`` - * you can also apply automatic fixing and formatting - using ``make fmt`` - -* Use version X.Y.Z-dev in dev mode. -* Use version X.Y.Z for releases. - -Versioning ----------- - -* Follow the `Semantic Versioning`_ guidelines. - -Release Process ---------------- - -* ``$EDITOR CHANGELOG.md``: update CHANGELOG with new notes, version, and date. -* commit changes -* ``$EDITOR lib/pyld/__about__.py``: update to release version and remove ``-dev`` - suffix. -* ``git commit CHANGELOG.md lib/pyld/__about__.py -m "Release {version}."`` -* ``git tag {version}`` -* ``$EDITOR lib/pyld/__about__.py``: update to next version and add ``-dev`` suffix. -* ``git commit lib/pyld/__about__.py -m "Start {next-version}."`` -* ``git push --tags`` - -To ensure a clean `package `_ upload to PyPI_, -use a clean checkout, and run the following: - -* For more info, look at the packaging - `guide `_. -* Setup an `API token `_. Recommend using a - specific "PyLD" token and set it up as a "repository" in your - `~/.pypirc `_ - for use in the upload command. -* The below builds and uploads a sdist and wheel. Adjust as needed depending - on how you manage and clean "dist/" dir files. -* ``git checkout {version}`` -* ``python3 -m build`` -* ``twine check dist/*`` -* ``twine upload -r PyLD dist/*`` - -Implementation Report Process ------------------------------ - -As of early 2020, the process to generate an EARL report for the official -`JSON-LD Processor Conformance`_ page is: - -* Run the tests on the ``json-ld-api`` and ``json-ld-framing`` test repos to - generate a ``.jsonld`` test report as explained in [README.rst](./README.rst#tests) -* Use the rdf_ tool to generate a ``.ttl``: - - * ``rdf serialize pyld-earl.jsonld --output-format turtle -o pyld-earl.ttl`` - -* Optionally follow the `report instructions`_ to generate the HTML report for - inspection. -* Submit a PR to the `json-ld-api repository`_ with at least the ``.ttl``: - -.. _JSON-LD Processor Conformance: https://w3c.github.io/json-ld-api/reports/ -.. _PEP 8 Style Guide: https://www.python.org/dev/peps/pep-0008/ -.. _Semantic Versioning: https://semver.org/ -.. _ruff: https://docs.astral.sh/ruff/ -.. _json-ld-api repository: https://github.com/w3c/json-ld-api/pulls -.. _rdf: https://rubygems.org/gems/rdf -.. _report instructions: https://github.com/w3c/json-ld-api/tree/master/reports -.. _PyPI: https://pypi.org/ diff --git a/MANIFEST.in b/MANIFEST.in index 8cd6fdad..96efd64f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -include README.rst README.txt LICENSE CHANGELOG.md +include README.md README.txt LICENSE CHANGELOG.md recursive-include lib/pyld/documentloader/frozen/bundled *.jsonld diff --git a/Makefile b/Makefile index 28da5f38..e5191c41 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,6 @@ -.PHONY: install test upgrade-submodules download-bundled-contexts +.PHONY: install test docs-install docs-build docs-serve upgrade-submodules download-bundled-contexts + +PORT ?= 8000 install: pip install -e . @@ -6,13 +8,23 @@ install: test: pytest --cov=pyld +docs-install: + python -m pip install --upgrade pip + pip install -r docs/requirements.txt + +docs-build: + mkdocs build --strict + +docs-serve: + mkdocs serve --dev-addr 127.0.0.1:$(PORT) + upgrade-submodules: git submodule update --remote --init --recursive download-bundled-contexts: python scripts/download_contexts.py -RUFF_TARGET = lib/pyld/*.py tests/*.py +RUFF_TARGET = lib/pyld/*.py tests/*.py docs_macros.py lint: ruff check $(RUFF_TARGET) diff --git a/README.md b/README.md new file mode 100644 index 00000000..b68a5e9c --- /dev/null +++ b/README.md @@ -0,0 +1,399 @@ +# PyLD + +Documentation: https://digitalbazaar.github.io/pyld/ + +## Introduction + +This library is an implementation of the JSON-LD specification in +[Python](https://www.python.org/). + +JSON, as specified in [RFC7159](http://tools.ietf.org/html/rfc7159), is a simple +language for representing objects on the Web. Linked Data is a way of describing +content across different documents or Web sites. Web resources are described +using IRIs, and typically are dereferencable entities that may be used to find +more information, creating a "Web of Knowledge". [JSON-LD](https://json-ld.org/) +is intended to be a simple publishing method for expressing not only Linked Data +in JSON, but for adding semantics to existing JSON. + +JSON-LD is designed as a light-weight syntax that can be used to express Linked +Data. It is primarily intended to be a way to express Linked Data in JavaScript +and other Web-based programming environments. It is also useful when building +interoperable Web Services and when storing Linked Data in JSON-based document +storage engines. It is practical and designed to be as simple as possible, +utilizing the large number of JSON parsers and existing code that is in use +today. It is designed to be able to express key-value pairs, RDF data, +[RDFa](http://www.w3.org/TR/rdfa-core/) data, +[Microformats](http://microformats.org/) data, and +[Microdata](http://www.w3.org/TR/microdata/). That is, it supports every major +Web-based structured data model in use today. + +The syntax does not require many applications to change their JSON, but easily +add meaning by adding context in a way that is either in-band or out-of-band. +The syntax is designed to not disturb already deployed systems running on JSON, +but provide a smooth migration path from JSON to JSON with added semantics. +Finally, the format is intended to be fast to parse, fast to generate, +stream-based and document-based processing compatible, and require a very small +memory footprint in order to operate. + +## Conformance + +This library aims to conform with the following W3C Recommendations: + +| Standard | Status | +| :--- | :--- | +| [JSON-LD 1.1](https://www.w3.org/TR/json-ld11/) | W3C Recommendation | +| [JSON-LD 1.1 Processing Algorithms and API](https://www.w3.org/TR/json-ld11-api/) | W3C Recommendation | +| [JSON-LD 1.1 Framing](https://www.w3.org/TR/json-ld11-framing/) | W3C Recommendation | +| [RDF Dataset Canonicalization](https://www.w3.org/TR/rdf-canon/) | W3C Recommendation | + + +The [`test +runner`](https://github.com/digitalbazaar/pyld/blob/master/tests/runtests.py) is +often updated to note or skip newer tests that are not yet supported. + +## Requirements + +* Python (3.10 or later) +* [Requests](http://docs.python-requests.org/) (optional) +* [aiohttp](https://aiohttp.readthedocs.io/) (optional) + +## Installation + +PyLD can be installed with a [pip](http://www.pip-installer.org/) +[package](https://pypi.org/project/PyLD/): + +```bash +pip install PyLD +``` + +Defining a dependency on pyld will not pull in +[Requests](http://docs.python-requests.org/) or +[aiohttp](https://aiohttp.readthedocs.io/). If you need one of these for a +[Document Loader](#document-loader) then either depend on the desired external library directly +or define the requirement as `PyLD[requests]` or `PyLD[aiohttp]`. + +## Quick Examples + +```python +from pyld import jsonld +import json + +doc = { + "http://schema.org/name": "Manu Sporny", + "http://schema.org/url": {"@id": "http://manu.sporny.org/"}, + "http://schema.org/image": {"@id": "http://manu.sporny.org/images/manu.png"} +} + +context = { + "name": "http://schema.org/name", + "homepage": {"@id": "http://schema.org/url", "@type": "@id"}, + "image": {"@id": "http://schema.org/image", "@type": "@id"} +} + +# compact a document according to a particular context +# see: https://json-ld.org/spec/latest/json-ld/#compacted-document-form +compacted = jsonld.compact(doc, context) + +print(json.dumps(compacted, indent=2)) +# Output: +# { +# "@context": {...}, +# "image": "http://manu.sporny.org/images/manu.png", +# "homepage": "http://manu.sporny.org/", +# "name": "Manu Sporny" +# } + +# compact using URLs +jsonld.compact('http://example.org/doc', 'http://example.org/context') + +# expand a document, removing its context +# see: https://json-ld.org/spec/latest/json-ld/#expanded-document-form +expanded = jsonld.expand(compacted) + +print(json.dumps(expanded, indent=2)) +# Output: +# [{ +# "http://schema.org/image": [{"@id": "http://manu.sporny.org/images/manu.png"}], +# "http://schema.org/name": [{"@value": "Manu Sporny"}], +# "http://schema.org/url": [{"@id": "http://manu.sporny.org/"}] +# }] + +# expand using URLs +jsonld.expand('http://example.org/doc') + +# flatten a document +# see: https://json-ld.org/spec/latest/json-ld/#flattened-document-form +flattened = jsonld.flatten(doc) +# all deep-level trees flattened to the top-level + +# frame a document +# see: https://json-ld.org/spec/latest/json-ld-framing/#introduction +framed = jsonld.frame(doc, frame) +# document transformed into a particular tree structure per the given frame + +# normalize a document using the RDF Dataset Normalization Algorithm +# (URDNA2015), see: https://www.w3.org/TR/rdf-canon/ +normalized = jsonld.normalize( + doc, {'algorithm': 'URDNA2015', 'format': 'application/n-quads'}) +# normalized is a string that is a canonical representation of the document +# that can be used for hashing, comparison, etc. +``` + +## Document Loader + +The default document loader for PyLD uses +[Requests](http://docs.python-requests.org/). In a production environment you +may want to setup a custom loader that, at a minimum, sets a timeout value. You +can also force requests to use https, set client certs, disable verification, or +set other Requests parameters. + +```python +jsonld.set_document_loader(jsonld.requests_document_loader(timeout=...)) +``` + +The factory remains the compatibility API, and the concrete class is also +available when class-based construction is preferred: + +```python +from pyld import RequestsDocumentLoader + +jsonld.set_document_loader(RequestsDocumentLoader(timeout=...)) +``` + +An asynchronous document loader using aiohttp is also available. Please note +that this document loader limits asynchronicity to fetching documents only. The +processing loops remain synchronous. + +```python +jsonld.set_document_loader(jsonld.aiohttp_document_loader(timeout=...)) +``` + +The concrete aiohttp loader class is available from `pyld` as well: + +```python +from pyld import AioHttpDocumentLoader + +jsonld.set_document_loader(AioHttpDocumentLoader(timeout=...)) +``` + +When no document loader is specified, the default loader is set to +[Requests](http://docs.python-requests.org/). If Requests is not available, the +loader is set to aiohttp. The fallback document loader is a dummy document +loader that raises an exception on every invocation. + +## Frozen Document Loader + +For air-gapped runs, reproducible builds, and security-hardened deployments that +must not perform any remote context fetches at all, PyLD ships +`FrozenDocumentLoader`: a class-based loader that serves only the URLs in its +`documents` allowlist and refuses everything else with +`JsonLdError(code='loading document failed')`. + +Instantiating with no arguments serves the curated `BUNDLED_CONTEXTS` set +(ActivityStreams, DID v1, Verifiable Credentials v1 and v2, Linked Data Security +v1/v2, Ed25519-2020, and JWS-2020). To extend the bundle with additional +pre-vetted contexts, pass a merged mapping: + +```python +from pyld import jsonld, FrozenDocumentLoader, BUNDLED_CONTEXTS + +loader = FrozenDocumentLoader(documents=dict( + BUNDLED_CONTEXTS, + **{'https://example.com/my-ctx': Path('contexts/my-ctx.jsonld')}, +)) +jsonld.expand(doc, options={'documentLoader': loader}) +``` + +This honors the W3C *JSON-LD Best Practices* recommendation that clients SHOULD +attempt to use a locally cached version of contexts (see +[§ Cache JSON-LD Contexts](https://w3c.github.io/json-ld-bp/#cache-json-ld-contexts)). +Refresh the bundled copies with `make download-bundled-contexts`. + +## Customizing the ContextLoader + +You can customize the way contexts are loaded and cached by passing an instance +of `ContextResolver`. The following example implements a loader with a prefilled +custom document cache and uses a custom LRU cache for resolved contexts: + +```python +from pyld.jsonld import compact, expand, set_document_loader, ContextResolver +import json +from cachetools import LRUCache + +# Load the Linked Art context from file-system +fh = open('linked-art.json') +js = json.load(fh) +fh.close() + +# Add to document cache +docCache = { + "https://linked.art/ns/v1/linked-art.json": { + "contextUrl": None, + "documentUrl": "https://linked.art/ns/v1/linked-art.json", + "document": js + } +} + +# Custom loader that uses the document cache +def load_document_and_cache(url, options={}): + if url in docCache: + return docCache[url] + doc = {"contextUrl": None, "documentUrl": url, "document": ""} + resp = requests.get(url) + doc["document"] = resp.json() + docCache[url] = doc + return doc + +# Set the custom loader as global document loader +set_document_loader(load_document_and_cache) +# Create custom context resolver with custom LRU cache and custom loader +resolved_context_cache = LRUCache(maxsize=1000) +resolver = ContextResolver(resolved_context_cache, load_document_and_cache) + +# Expand JSON-LD document using custom context resolver +input = {"@context":"https://linked.art/ns/v1/linked-art.json", "id": "tag:foo", "type": "Person"} +output = expand(input, options={'contextResolver': resolver}) +``` + +It is also possible to change the maximum number of times that the loader +recursively fetches contexts, by passing the `max_context_urls` parameter: + +```python +resolver = ContextResolver(resolved_context_cache, load_document_and_cache, max_context_urls=20) +# Or you can do... +# resolver = ContextResolver(resolved_context_cache, load_document_and_cache) +# resolver.max_context_urls = 20 +output = expand(input, options={'contextResolver': resolver}) +``` + +## Handling ignored properties during JSON-LD expansion + +If a property in a JSON-LD document does not map to an absolute IRI then it is +ignored. You can customize this behaviour by passing a customizable handler to +`on_property_dropped` parameter of `jsonld.expand()`. + +For example, you can introduce a strict mode by raising a ValueError on every +dropped property: + +```python +def raise_this(value): + raise ValueError(value) + +jsonld.expand(doc, None, on_property_dropped=raise_this) +``` + +## Commercial Support + +Commercial support for this library is available upon request from [`Digital +Bazaar`](mailto:support@digitalbazaar.com). + +## Source + +The source code for the Python implementation of the JSON-LD API is available +at: + +[https://github.com/digitalbazaar/pyld](https://github.com/digitalbazaar/pyld) + +## Tests + +This library includes a sample testing utility which may be used to verify that +changes to the processor maintain the correct output. + +To run the sample tests you will need to get the test suite files, which by +default, are stored in the `specifications/` folder. The test suites can be +obtained by either using git submodules or by cloning them manually. + +### Using git submodules + +The test suites are included as git submodules to ensure versions are in sync. +When cloning the repository, use the `--recurse-submodules` flag to +automatically clone the submodules. If you have cloned the repository without +the submodules, you can initialize them with the following commands: + +```bash +git submodule init +git submodule update +``` + +### Cloning manually + +You can also avoid using git submodules by manually cloning the `json-ld-api`, +`json-ld-framing`, and `normalization` repositories hosted on GitHub using the +following commands: + +```bash +git clone https://github.com/w3c/json-ld-api ./specifications/json-ld-api +git clone https://github.com/w3c/json-ld-framing ./specifications/json-ld-framing +git clone https://github.com/json-ld/normalization ./specifications/normalization +``` + +Note that you can clone these repositories into any location you wish; however, +if you do not clone them into the default `specifications/` folder, you will +need to provide the paths to the test runner as arguments when running the +tests, as explained below. + +### Running the sample test suites and unit tests using pytest + +If the suites repositories are available in the `specifications/` folder of the +PyLD source directory, then all unittests, including the sample test suites, can +be run with `pytest`: + +```bash +pytest +``` + +If you wish to store the test suites in a different location than the default +`specifications/` folder, or you want to test individual manifest `.jsonld` +files or directories containing a `manifest.jsonld`, then you can supply these +files or directories as arguments: + +```bash +# use: pytest --tests=TEST_PATH [--tests=TEST_PATH...] +pytest --tests=./specifications/json-ld-api/tests +``` + +The test runner supports different document loaders by setting `--loader +requests` or `--loader aiohttp`. The default document loader is set to +[Requests](http://docs.python-requests.org/). + +```bash +pytest --loader=requests --tests=./specifications/json-ld-api/tests +``` + +An EARL report can be generated using the `--earl` option. + +```bash +pytest --earl=./earl-report.json +``` + +### Running the sample test suites using the original test runner + +You can also run the JSON-LD test suites using the original test runner script +provided: + +```bash +python tests/runtests.py +``` + +If you wish to store the test suites in a different location than the default +`specifications/` folder, or you want to test individual manifest `.jsonld` +files or directories containing a `manifest.jsonld`, then you can supply these +files or directories as arguments: + +```bash +python tests/runtests.py TEST_PATH [TEST_PATH...] +``` + +The test runner supports different document loaders by setting `-l requests` or +`-l aiohttp`. The default document loader is set to +[Requests](http://docs.python-requests.org/). + +```bash +python tests/runtests.py -l requests ./specifications/json-ld-api/tests +``` + +An EARL report can be generated using the `-e` or `--earl` option. + +```bash +python tests/runtests.py -e ./earl-report.json +``` \ No newline at end of file diff --git a/README.rst b/README.rst deleted file mode 100644 index 1fe27f40..00000000 --- a/README.rst +++ /dev/null @@ -1,446 +0,0 @@ -PyLD -==== - -.. image:: https://travis-ci.org/digitalbazaar/pyld.png?branch=master - :target: https://travis-ci.org/digitalbazaar/pyld - :alt: Build Status - -Introduction ------------- - -This library is an implementation of the JSON-LD_ specification in Python_. - -JSON, as specified in RFC7159_, is a simple language for representing -objects on the Web. Linked Data is a way of describing content across -different documents or Web sites. Web resources are described using -IRIs, and typically are dereferencable entities that may be used to find -more information, creating a "Web of Knowledge". JSON-LD_ is intended -to be a simple publishing method for expressing not only Linked Data in -JSON, but for adding semantics to existing JSON. - -JSON-LD is designed as a light-weight syntax that can be used to express -Linked Data. It is primarily intended to be a way to express Linked Data -in JavaScript and other Web-based programming environments. It is also -useful when building interoperable Web Services and when storing Linked -Data in JSON-based document storage engines. It is practical and -designed to be as simple as possible, utilizing the large number of JSON -parsers and existing code that is in use today. It is designed to be -able to express key-value pairs, RDF data, RDFa_ data, -Microformats_ data, and Microdata_. That is, it supports every -major Web-based structured data model in use today. - -The syntax does not require many applications to change their JSON, but -easily add meaning by adding context in a way that is either in-band or -out-of-band. The syntax is designed to not disturb already deployed -systems running on JSON, but provide a smooth migration path from JSON -to JSON with added semantics. Finally, the format is intended to be fast -to parse, fast to generate, stream-based and document-based processing -compatible, and require a very small memory footprint in order to operate. - -Conformance ------------ - -This library aims to conform with the following: - -- `JSON-LD 1.1 `_, - W3C Candidate Recommendation, - 2019-12-12 or `newer `_ -- `JSON-LD 1.1 Processing Algorithms and API `_, - W3C Candidate Recommendation, - 2019-12-12 or `newer `_ -- `JSON-LD 1.1 Framing `_, - W3C Candidate Recommendation, - 2019-12-12 or `newer `_ -- Working Group `test suite `_ - -The `test runner`_ is often updated to note or skip newer tests that are not -yet supported. - -Requirements ------------- - -- Python_ (3.10 or later) -- Requests_ (optional) -- aiohttp_ (optional, Python 3.5 or later) - -Installation ------------- - -PyLD can be installed with a pip_ `package `_ - -.. code-block:: bash - - pip install PyLD - -Defining a dependency on pyld will not pull in Requests_ or aiohttp_. If you -need one of these for a `Document Loader`_ then either depend on the desired -external library directly or define the requirement as ``PyLD[requests]`` or -``PyLD[aiohttp]``. - -Quick Examples --------------- - -.. code-block:: Python - - from pyld import jsonld - import json - - doc = { - "http://schema.org/name": "Manu Sporny", - "http://schema.org/url": {"@id": "http://manu.sporny.org/"}, - "http://schema.org/image": {"@id": "http://manu.sporny.org/images/manu.png"} - } - - context = { - "name": "http://schema.org/name", - "homepage": {"@id": "http://schema.org/url", "@type": "@id"}, - "image": {"@id": "http://schema.org/image", "@type": "@id"} - } - - # compact a document according to a particular context - # see: https://json-ld.org/spec/latest/json-ld/#compacted-document-form - compacted = jsonld.compact(doc, context) - - print(json.dumps(compacted, indent=2)) - # Output: - # { - # "@context": {...}, - # "image": "http://manu.sporny.org/images/manu.png", - # "homepage": "http://manu.sporny.org/", - # "name": "Manu Sporny" - # } - - # compact using URLs - jsonld.compact('http://example.org/doc', 'http://example.org/context') - - # expand a document, removing its context - # see: https://json-ld.org/spec/latest/json-ld/#expanded-document-form - expanded = jsonld.expand(compacted) - - print(json.dumps(expanded, indent=2)) - # Output: - # [{ - # "http://schema.org/image": [{"@id": "http://manu.sporny.org/images/manu.png"}], - # "http://schema.org/name": [{"@value": "Manu Sporny"}], - # "http://schema.org/url": [{"@id": "http://manu.sporny.org/"}] - # }] - - # expand using URLs - jsonld.expand('http://example.org/doc') - - # flatten a document - # see: https://json-ld.org/spec/latest/json-ld/#flattened-document-form - flattened = jsonld.flatten(doc) - # all deep-level trees flattened to the top-level - - # frame a document - # see: https://json-ld.org/spec/latest/json-ld-framing/#introduction - framed = jsonld.frame(doc, frame) - # document transformed into a particular tree structure per the given frame - - # normalize a document using the RDF Dataset Normalization Algorithm - # (URDNA2015), see: https://www.w3.org/TR/rdf-canon/ - normalized = jsonld.normalize( - doc, {'algorithm': 'URDNA2015', 'format': 'application/n-quads'}) - # normalized is a string that is a canonical representation of the document - # that can be used for hashing, comparison, etc. - -Document Loader ---------------- - -The default document loader for PyLD uses Requests_. In a production -environment you may want to setup a custom loader that, at a minimum, sets a -timeout value. You can also force requests to use https, set client certs, -disable verification, or set other Requests_ parameters. - -.. code-block:: Python - - jsonld.set_document_loader(jsonld.requests_document_loader(timeout=...)) - -The factory remains the compatibility API, and the concrete class is also -available when class-based construction is preferred: - -.. code-block:: Python - - from pyld import RequestsDocumentLoader - - jsonld.set_document_loader(RequestsDocumentLoader(timeout=...)) - -An asynchronous document loader using aiohttp_ is also available. Please note -that this document loader limits asynchronicity to fetching documents only. -The processing loops remain synchronous. - -.. code-block:: Python - - jsonld.set_document_loader(jsonld.aiohttp_document_loader(timeout=...)) - -The concrete aiohttp loader class is available from ``pyld`` as well: - -.. code-block:: Python - - from pyld import AioHttpDocumentLoader - - jsonld.set_document_loader(AioHttpDocumentLoader(timeout=...)) - -When no document loader is specified, the default loader is set to Requests_. -If Requests_ is not available, the loader is set to aiohttp_. The fallback -document loader is a dummy document loader that raises an exception on every -invocation. - -Frozen Document Loader -###################### - -For air-gapped runs, reproducible builds, and security-hardened deployments -that must not perform any remote context fetches at all, PyLD ships -``FrozenDocumentLoader``: a class-based loader that serves only the URLs in -its ``documents`` allowlist and refuses everything else with -``JsonLdError(code='loading document failed')``. - -Instantiating with no arguments serves the curated ``BUNDLED_CONTEXTS`` set -(ActivityStreams, DID v1, Verifiable Credentials v1 and v2, Linked Data -Security v1/v2, Ed25519-2020, and JWS-2020). To extend the bundle with -additional pre-vetted contexts, pass a merged mapping: - -.. code-block:: Python - - from pyld import jsonld, FrozenDocumentLoader, BUNDLED_CONTEXTS - - loader = FrozenDocumentLoader(documents=dict( - BUNDLED_CONTEXTS, - **{'https://example.com/my-ctx': Path('contexts/my-ctx.jsonld')}, - )) - jsonld.expand(doc, options={'documentLoader': loader}) - -This honors the W3C *JSON-LD Best Practices* recommendation that clients -SHOULD attempt to use a locally cached version of contexts (see -`§ Cache JSON-LD Contexts `_). -Refresh the bundled copies with ``make download-bundled-contexts``. - -Customizing the ContextLoader ------------------------------ - -You can customize the way contexts are loaded and cached by passing an instance -of ``ContextResolver``. The following example implements a loader with a -prefilled custom document cache and uses a custom LRU cache for resolved -contexts: - -.. code-block:: Python - - from pyld.jsonld import compact, expand, set_document_loader, ContextResolver - import json - from cachetools import LRUCache - - # Load the Linked Art context from file-system - fh = open('linked-art.json') - js = json.load(fh) - fh.close() - - # Add to document cache - docCache = { - "https://linked.art/ns/v1/linked-art.json": { - "contextUrl": None, - "documentUrl": "https://linked.art/ns/v1/linked-art.json", - "document": js - } - } - - # Custom loader that uses the document cache - def load_document_and_cache(url, options={}): - if url in docCache: - return docCache[url] - doc = {"contextUrl": None, "documentUrl": url, "document": ""} - resp = requests.get(url) - doc["document"] = resp.json() - docCache[url] = doc - return doc - - # Set the custom loader as global document loader - set_document_loader(load_document_and_cache) - # Create custom context resolver with custom LRU cache and custom loader - resolved_context_cache = LRUCache(maxsize=1000) - resolver = ContextResolver(resolved_context_cache, load_document_and_cache) - - # Expand JSON-LD document using custom context resolver - input = {"@context":"https://linked.art/ns/v1/linked-art.json", "id": "tag:foo", "type": "Person"} - output = expand(input, options={'contextResolver': resolver}) - - -It is also possible to change the maximum number of times that the loader recusively fetches contexts, -by passing the ``max_context_urls`` parameter: - -.. code-block:: Python - - resolver = ContextResolver(resolved_context_cache, load_document_and_cache, max_context_urls=20) - # Or you can do... - # resolver = ContextResolver(resolved_context_cache, load_document_and_cache) - # resolver.max_context_urls = 20 - output = expand(input, options={'contextResolver': resolver}) - - -Handling ignored properties during JSON-LD expansion ----------------------------------------------------- - -If a property in a JSON-LD document does not map to an absolute IRI then it is -ignored. You can customize this behaviour by passing a customizable handler to -`on_property_dropped` parameter of `jsonld.expand()`. - -For example, you can introduce a strict mode by raising a ValueError on every -dropped property: - -.. code-block:: Python - - def raise_this(value): - raise ValueError(value) - - jsonld.expand(doc, None, on_property_dropped=raise_this) - -Commercial Support ------------------- - -Commercial support for this library is available upon request from -`Digital Bazaar`_: support@digitalbazaar.com. - -Source ------- - -The source code for the Python implementation of the JSON-LD API -is available at: - -https://github.com/digitalbazaar/pyld - -Tests ------ - -This library includes a sample testing utility which may be used to verify -that changes to the processor maintain the correct output. - -To run the sample tests you will need to get the test suite files, which -by default, are stored in the `specifications/` folder. -The test suites can be obtained by either using git submodules or by cloning -them manually. - -Using git submodules -#################### - -The test suites are included as git submodules to ensure versions are in sync. -When cloning the repository, use the ``--recurse-submodules`` flag to -automatically clone the submodules. -If you have cloned the repository without the submodules, you can initialize -them with the following commands: - -.. code-block:: bash - - git submodule init - git submodule update - -Cloning manually -#################### - -You can also avoid using git submodules by manually cloning -the ``json-ld-api``, ``json-ld-framing``, and ``normalization`` repositories -hosted on GitHub using the following commands: - -.. code-block:: bash - - git clone https://github.com/w3c/json-ld-api ./specifications/json-ld-api - git clone https://github.com/w3c/json-ld-framing ./specifications/json-ld-framing - git clone https://github.com/json-ld/normalization ./specifications/normalization - -Note that you can clone these repositories into any location you wish; however, -if you do not clone them into the default ``specifications/`` folder, you will -need to provide the paths to the test runner as arguments when running the -tests, as explained below - -Running the sample test suites and unittests using pytest -######################################################### - -If the suites repositories are available in the `specifications/` folder of the -PyLD source directory, then all unittests, including the sample test suites, -can be run with `pytest`: - -.. code-block:: bash - - pytest - -If you wish to store the test suites in a different location than the default -``specifications/`` folder, or you want to test individual manifest ``.jsonld`` -files or directories containing a ``manifest.jsonld``, then you can supply -these files or directories as arguments: - -.. code-block:: bash - - # use: pytest --tests=TEST_PATH [--tests=TEST_PATH...] - pytest --tests=./specifications/json-ld-api/tests - -The test runner supports different document loaders by setting -``--loader requests`` or ``--loader aiohttp``. The default document loader is -set to Requests_. - -.. code-block:: bash - - pytest --loader=requests --tests=./specifications/json-ld-api/tests - -An EARL report can be generated using the ``--earl`` option. - -.. code-block:: bash - - pytest --earl=./earl-report.json - -Running the sample test suites using the original test runner -############################################################# - -You can also run the JSON-LD test suites using the original test runner script -provided: - -.. code-block:: bash - - python tests/runtests.py - -If you wish to store the test suites in a different location than the default -``specifications/`` folder, or you want to test individual manifest ``.jsonld`` -files or directories containing a ``manifest.jsonld``, then you can supply -these files or directories as arguments: - -.. code-block:: bash - - python tests/runtests.py TEST_PATH [TEST_PATH...] - -The test runner supports different document loaders by setting ``-l requests`` -or ``-l aiohttp``. The default document loader is set to Requests_. - -.. code-block:: bash - - python tests/runtests.py -l requests ./specifications/json-ld-api/tests - -An EARL report can be generated using the ``-e`` or ``--earl`` option. - -.. code-block:: bash - - python tests/runtests.py -e ./earl-report.json - - -.. _Digital Bazaar: https://digitalbazaar.com/ - -.. _JSON-LD WG 1.1 API: https://www.w3.org/TR/json-ld11-api/ -.. _JSON-LD WG 1.1 Framing: https://www.w3.org/TR/json-ld11-framing/ -.. _JSON-LD WG 1.1: https://www.w3.org/TR/json-ld11/ - -.. _JSON-LD WG API latest: https://w3c.github.io/json-ld-api/ -.. _JSON-LD WG Framing latest: https://w3c.github.io/json-ld-framing/ -.. _JSON-LD WG latest: https://w3c.github.io/json-ld-syntax/ - -.. _JSON-LD Benchmarks: https://json-ld.org/benchmarks/ -.. _JSON-LD WG: https://www.w3.org/2018/json-ld-wg/ -.. _JSON-LD: https://json-ld.org/ -.. _Microdata: http://www.w3.org/TR/microdata/ -.. _Microformats: http://microformats.org/ -.. _Python: https://www.python.org/ -.. _Requests: http://docs.python-requests.org/ -.. _aiohttp: https://aiohttp.readthedocs.io/ -.. _RDFa: http://www.w3.org/TR/rdfa-core/ -.. _RFC7159: http://tools.ietf.org/html/rfc7159 -.. _WG test suite: https://github.com/w3c/json-ld-api/tree/master/tests -.. _errata: http://www.w3.org/2014/json-ld-errata -.. _pip: http://www.pip-installer.org/ -.. _test runner: https://github.com/digitalbazaar/pyld/blob/master/tests/runtests.py -.. _test suite: https://github.com/json-ld/json-ld.org/tree/master/test-suite diff --git a/README.txt b/README.txt index 92cacd28..42061c01 120000 --- a/README.txt +++ b/README.txt @@ -1 +1 @@ -README.rst \ No newline at end of file +README.md \ No newline at end of file diff --git a/docs/.pages b/docs/.pages new file mode 100644 index 00000000..1b153383 --- /dev/null +++ b/docs/.pages @@ -0,0 +1,7 @@ +nav: + - Quickstart: + - index.md + - installation.md + - conformance.md + - Reference: + - reference diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 88452c53..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,136 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build -HOST = 127.0.0.1 -PORT = 8000 - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml serve pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " serve to serve HTML files with auto rebuild" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -serve: - sphinx-autobuild -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html --host $(HOST) --port $(PORT) - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PyLD.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyLD.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/PyLD" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PyLD" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - make -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/assets/cover.png b/docs/assets/cover.png new file mode 100644 index 00000000..dd65e135 Binary files /dev/null and b/docs/assets/cover.png differ diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index c4bf2b85..00000000 --- a/docs/conf.py +++ /dev/null @@ -1,223 +0,0 @@ -# -*- coding: utf-8 -*- -# -# PyLD documentation build configuration file, created by -# sphinx-quickstart on Mon Aug 29 15:25:28 2011. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys, os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - -current_path = os.path.abspath(os.path.dirname(__file__)) -path = os.path.join(current_path, '..') - -sys.path[0:0] = [ - os.path.join(path, 'lib'), -] - -# -- General configuration ----------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'PyLD' -copyright = u'2011, Digital Bazaar' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '0.0' -# The full version, including alpha/beta/rc tags. -release = '0.0.1' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'default' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'PyLDdoc' - - -# -- Options for LaTeX output -------------------------------------------------- - -# The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' - -# The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'PyLD.tex', u'PyLD Documentation', - u'Digital Bazaar', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Additional stuff for the LaTeX preamble. -#latex_preamble = '' - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'pyld', u'PyLD Documentation', - [u'Digital Bazaar'], 1) -] diff --git a/docs/conformance.md b/docs/conformance.md new file mode 100644 index 00000000..bb8335ba --- /dev/null +++ b/docs/conformance.md @@ -0,0 +1,21 @@ +--- +hide: [toc] +--- + +# :fontawesome-solid-legal: Conformance + +PyLD aims to conform with: + +- [JSON-LD 1.1](https://www.w3.org/TR/json-ld11/) +- [JSON-LD 1.1 Processing Algorithms and API](https://www.w3.org/TR/json-ld11-api/) +- [JSON-LD 1.1 Framing](https://www.w3.org/TR/json-ld11-framing/) +- [RDF Dataset Canonicalization (URDNA2015)](https://www.w3.org/TR/rdf-canon/) +- [RDF Dataset Normalization (URGNA2012)](https://www.w3.org/TR/rdf-graph-normalization/) +- The JSON-LD Working Group [test suite](https://github.com/w3c/json-ld-api/tree/master/tests) + +The test runner is updated over time to note or skip newer tests that are not +yet supported. + +## Skipped tests + +{{ skipped_tests_table() }} diff --git a/docs/examples/document_loaders/aiohttp_class.py b/docs/examples/document_loaders/aiohttp_class.py new file mode 100644 index 00000000..ba3399be --- /dev/null +++ b/docs/examples/document_loaders/aiohttp_class.py @@ -0,0 +1,12 @@ +import json + +from pyld import AioHttpDocumentLoader, jsonld + +doc = { + "@context": {"name": "http://schema.org/name"}, + "name": "Earth", +} + +loader = AioHttpDocumentLoader(timeout=10) +result = jsonld.expand(doc, options={"documentLoader": loader}) +print(json.dumps(result, indent=2)) diff --git a/docs/examples/document_loaders/aiohttp_extra_kwargs.py b/docs/examples/document_loaders/aiohttp_extra_kwargs.py new file mode 100644 index 00000000..ba3399be --- /dev/null +++ b/docs/examples/document_loaders/aiohttp_extra_kwargs.py @@ -0,0 +1,12 @@ +import json + +from pyld import AioHttpDocumentLoader, jsonld + +doc = { + "@context": {"name": "http://schema.org/name"}, + "name": "Earth", +} + +loader = AioHttpDocumentLoader(timeout=10) +result = jsonld.expand(doc, options={"documentLoader": loader}) +print(json.dumps(result, indent=2)) diff --git a/docs/examples/document_loaders/aiohttp_secure.py b/docs/examples/document_loaders/aiohttp_secure.py new file mode 100644 index 00000000..9da3a91c --- /dev/null +++ b/docs/examples/document_loaders/aiohttp_secure.py @@ -0,0 +1,12 @@ +import json + +from pyld import AioHttpDocumentLoader, jsonld + +doc = { + "@context": {"name": "http://schema.org/name"}, + "name": "Earth", +} + +loader = AioHttpDocumentLoader(secure=True, timeout=10) +result = jsonld.expand(doc, options={"documentLoader": loader}) +print(json.dumps(result, indent=2)) diff --git a/docs/examples/document_loaders/custom_document_loader.py b/docs/examples/document_loaders/custom_document_loader.py new file mode 100644 index 00000000..3dba37d9 --- /dev/null +++ b/docs/examples/document_loaders/custom_document_loader.py @@ -0,0 +1,27 @@ +import json + +from pyld import DocumentLoader, jsonld + +DOCUMENT_CACHE = { + "context://my-app/vocab": { + "contentType": "application/ld+json", + "contextUrl": None, + "documentUrl": "context://my-app/vocab", + "document": {"@context": {"name": "https://schema.org/name"}}, + } +} + + +class ExampleDocumentLoader(DocumentLoader): + def __call__(self, url, options): + return DOCUMENT_CACHE[url] + + +doc = { + "@context": "context://my-app/vocab", + "name": "Earth", +} + +loader = ExampleDocumentLoader() +result = jsonld.expand(doc, options={"documentLoader": loader}) +print(json.dumps(result, indent=2)) diff --git a/docs/examples/document_loaders/frozen_default.py b/docs/examples/document_loaders/frozen_default.py new file mode 100644 index 00000000..b6683e45 --- /dev/null +++ b/docs/examples/document_loaders/frozen_default.py @@ -0,0 +1,12 @@ +import json + +from pyld import FrozenDocumentLoader, jsonld + +doc = { + "@context": {"name": "http://schema.org/name"}, + "name": "Earth", +} + +loader = FrozenDocumentLoader() +result = jsonld.expand(doc, options={"documentLoader": loader}) +print(json.dumps(result, indent=2)) diff --git a/docs/examples/document_loaders/frozen_extend.py b/docs/examples/document_loaders/frozen_extend.py new file mode 100644 index 00000000..938786ec --- /dev/null +++ b/docs/examples/document_loaders/frozen_extend.py @@ -0,0 +1,22 @@ +import json + +from pyld import BUNDLED_CONTEXTS, FrozenDocumentLoader, jsonld + +loader = FrozenDocumentLoader( + documents=dict( + BUNDLED_CONTEXTS, + **{ + "https://example.com/context": { + "@context": {"name": "https://schema.org/name"} + } + }, + ) +) + +doc = { + "@context": "https://example.com/context", + "name": "Earth", +} + +result = jsonld.expand(doc, options={"documentLoader": loader}) +print(json.dumps(result, indent=2)) diff --git a/docs/examples/document_loaders/requests_extra_kwargs.py b/docs/examples/document_loaders/requests_extra_kwargs.py new file mode 100644 index 00000000..8905908d --- /dev/null +++ b/docs/examples/document_loaders/requests_extra_kwargs.py @@ -0,0 +1,16 @@ +import json + +from pyld import RequestsDocumentLoader, jsonld + +doc = { + "@context": {"name": "http://schema.org/name"}, + "name": "Earth", +} + +loader = RequestsDocumentLoader( + timeout=10, + verify=True, + cert=("client.crt", "client.key"), +) +result = jsonld.expand(doc, options={"documentLoader": loader}) +print(json.dumps(result, indent=2)) diff --git a/docs/examples/document_loaders/requests_secure.py b/docs/examples/document_loaders/requests_secure.py new file mode 100644 index 00000000..44d5faed --- /dev/null +++ b/docs/examples/document_loaders/requests_secure.py @@ -0,0 +1,12 @@ +import json + +from pyld import RequestsDocumentLoader, jsonld + +doc = { + "@context": {"name": "http://schema.org/name"}, + "name": "Earth", +} + +loader = RequestsDocumentLoader(secure=True, timeout=10) +result = jsonld.expand(doc, options={"documentLoader": loader}) +print(json.dumps(result, indent=2)) diff --git a/docs/examples/document_loaders/requests_timeout.py b/docs/examples/document_loaders/requests_timeout.py new file mode 100644 index 00000000..818b9dcb --- /dev/null +++ b/docs/examples/document_loaders/requests_timeout.py @@ -0,0 +1,12 @@ +import json + +from pyld import RequestsDocumentLoader, jsonld + +doc = { + "@context": {"name": "http://schema.org/name"}, + "name": "Earth", +} + +loader = RequestsDocumentLoader(timeout=10) +result = jsonld.expand(doc, options={"documentLoader": loader}) +print(json.dumps(result, indent=2)) diff --git a/docs/examples/earth.py b/docs/examples/earth.py new file mode 100644 index 00000000..1a278789 --- /dev/null +++ b/docs/examples/earth.py @@ -0,0 +1,11 @@ +from pyld import jsonld + +doc = { + "@context": { + "name": "http://schema.org/name", + }, + "@id": "http://dbpedia.org/resource/Earth", + "name": "Earth", +} + +print(jsonld.to_rdf(doc, {"format": "application/n-quads"})) diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..d3c0fe73 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,41 @@ +--- +hide: [toc] +--- + +# :material-graph-outline: PyLD + +[![Main CI](https://github.com/digitalbazaar/pyld/actions/workflows/main.yaml/badge.svg)](https://github.com/digitalbazaar/pyld/actions/workflows/main.yaml) +[![PyPI](https://img.shields.io/pypi/v/PyLD.svg)](https://pypi.org/project/PyLD/) + +PyLD is a Python implementation of the [JSON-LD](https://json-ld.org/) processor API. + +![](assets/cover.png) + +JSON-LD is a lightweight syntax for expressing Linked Data in JSON. It lets +applications add meaning to existing JSON documents with in-band or out-of-band +contexts, while keeping the document shape practical for web APIs, JavaScript, +and JSON document stores. + +{{ example('earth.py') }} + +## :fontawesome-solid-people-line: Maintainers + +
+ +- [![Miel Vander Sande](https://github.com/mielvds.png?s=128)](https://github.com/mielvds) + + __Miel Vander Sande__ + + --- + + :fontawesome-brands-github:{ .middle } [`@mielvds`](https://github.com/mielvds) + +- [![Anatoly Scherbakov](https://github.com/anatoly-scherbakov.png?s=128)](https://github.com/anatoly-scherbakov) + + __Anatoly Scherbakov__ + + --- + + :fontawesome-brands-github:{ .middle } [`@anatoly-scherbakov`](https://github.com/anatoly-scherbakov) + +
diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 89b11570..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,68 +0,0 @@ -.. PyLD documentation master file, created by - sphinx-quickstart on Mon Aug 29 15:25:28 2011. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to PyLD! -================ - -PyLD is a `Python`_ implementation of a `JSON-LD`_ processor. - - -API Reference -------------- - -.. toctree:: - :maxdepth: 2 - -.. module:: pyld.jsonld -.. autofunction:: compact -.. autofunction:: expand -.. autofunction:: flatten -.. autofunction:: frame -.. autofunction:: normalize - -.. module:: pyld -.. autoclass:: DocumentLoader - :members: -.. autoclass:: RequestsDocumentLoader - :members: -.. autoclass:: AioHttpDocumentLoader - :members: -.. autoclass:: FrozenDocumentLoader - :members: -.. autodata:: BUNDLED_CONTEXTS - -Indices and tables ------------------- - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - -Requirements ------------- - -PyLD is compatible with `Python`_ 2.5 and newer. - -Credits -------- - -Thanks to `Digital Bazaar`_, the JavaScript JSON-LD parser, and the `JSON-LD`_ community. - -Contribute ----------- - -Source code is available: - - https://github.com/digitalbazaar/pyld - -License -------- - -PyLD is licensed under a `BSD 3-Clause license`_. - -.. _JSON-LD: https://json-ld.org/ -.. _Digital Bazaar: https://digitalbazaar.com/ -.. _Python: https://www.python.org/ -.. _BSD 3-Clause License: https://opensource.org/licenses/BSD-3-Clause diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 00000000..7659708f --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,41 @@ +# :simple-pypi: Installation + +Install PyLD from PyPI: + +```bash +pip install PyLD +``` + +PyLD's core package does not install `requests` or `aiohttp` automatically. If +your application needs one of the built-in remote document loaders, install the +matching extra: + +```bash +pip install "PyLD[requests]" +pip install "PyLD[aiohttp]" +``` + +You can also depend on `requests` or `aiohttp` directly if your project already +manages those dependencies. + +## Development Install + +From a local checkout: + +```bash +pip install -e . +``` + +Run the project tests with: + +```bash +pytest +``` + +The JSON-LD specification test suites are stored under `specifications/` and are +usually initialized as git submodules: + +```bash +git submodule init +git submodule update +``` diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index a24b446a..00000000 --- a/docs/make.bat +++ /dev/null @@ -1,170 +0,0 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. changes to make an overview over all changed/added/deprecated items - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PyLD.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PyLD.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -:end diff --git a/docs/reference/document-loaders/aiohttp.md b/docs/reference/document-loaders/aiohttp.md new file mode 100644 index 00000000..9a7a7f86 --- /dev/null +++ b/docs/reference/document-loaders/aiohttp.md @@ -0,0 +1,22 @@ +# :material-sync: `AioHttpDocumentLoader` + +`AioHttpDocumentLoader` retrieves JSON-LD documents with `aiohttp`. + +{{ example('document_loaders/aiohttp_class.py', output_syntax='json') }} + +This loader uses asynchronous fetching internally, but JSON-LD processing itself +remains synchronous. + +Use `secure=True` to require HTTPS URLs: + +{{ example('document_loaders/aiohttp_secure.py', output_syntax='json') }} + +Extra keyword arguments are forwarded to `aiohttp` request calls: + +{{ example('document_loaders/aiohttp_extra_kwargs.py', output_syntax='json') }} + +Install the optional dependency with: + +```bash +pip install "PyLD[aiohttp]" +``` diff --git a/docs/reference/document-loaders/custom.md b/docs/reference/document-loaders/custom.md new file mode 100644 index 00000000..e0b7c940 --- /dev/null +++ b/docs/reference/document-loaders/custom.md @@ -0,0 +1,8 @@ +# :material-code-braces: Custom Document Loaders + +Subclass `DocumentLoader` and implement `__call__` to return a remote document +mapping with `contentType`, `contextUrl`, `documentUrl`, and `document` keys. +Custom schemes such as `context://` cannot be fetched over HTTP, so a custom +loader is required to resolve them. + +{{ example('document_loaders/custom_document_loader.py', output_syntax='json') }} diff --git a/docs/reference/document-loaders/frozen.md b/docs/reference/document-loaders/frozen.md new file mode 100644 index 00000000..c22012c6 --- /dev/null +++ b/docs/reference/document-loaders/frozen.md @@ -0,0 +1,24 @@ +# :material-snowflake: `FrozenDocumentLoader` + +`FrozenDocumentLoader` serves only URLs in an allowlist and refuses all other +document loads. It is intended for air-gapped runs, reproducible builds, and +deployments that must avoid remote context fetching. + +With no arguments, the loader serves the curated `BUNDLED_CONTEXTS` mapping: + +{{ example('document_loaders/frozen_default.py', output_syntax='json') }} + +## Bundled Contexts + +{{ bundled_contexts_table() }} + +Extend the bundled mapping with additional vetted contexts: + +{{ example('document_loaders/frozen_extend.py', output_syntax='json') }} + +The `documents` mapping may contain parsed JSON-LD dictionaries or +`pathlib.Path` instances pointing to JSON files. Path entries are read lazily +and cached after the first request. + +Any URL outside the allowlist raises `JsonLdError` with code +`loading document failed`. diff --git a/docs/reference/document-loaders/index.md b/docs/reference/document-loaders/index.md new file mode 100644 index 00000000..c784d2bc --- /dev/null +++ b/docs/reference/document-loaders/index.md @@ -0,0 +1,41 @@ +# :material-file-download-outline: Document Loaders + +Document loaders retrieve remote JSON-LD documents and contexts. PyLD ships +class-based loaders for common cases and supports custom subclasses of +`DocumentLoader`. + +
+ +- [:material-cloud-download:{ .lg .middle } `RequestsDocumentLoader`](requests.md) + + --- + + Synchronous remote document loading with `requests`. + +- [:material-sync:{ .lg .middle } `AioHttpDocumentLoader`](aiohttp.md) + + --- + + Asynchronous fetching with `aiohttp` while JSON-LD processing stays + synchronous. + +- [:material-snowflake:{ .lg .middle } `FrozenDocumentLoader`](frozen.md) + + --- + + Serve only documents from an allowlist for air-gapped or reproducible runs. + +- [:material-code-braces:{ .lg .middle } __Custom Document Loaders__](custom.md) + + --- + + Subclass `DocumentLoader` for application-specific loading logic. + +
+ +## Default Document Loader + +The default document loader is selected at import time. PyLD uses +`RequestsDocumentLoader` if `requests` is available, falls back to +`AioHttpDocumentLoader` if `aiohttp` is available, and otherwise installs a +dummy loader that raises when invoked. diff --git a/docs/reference/document-loaders/requests.md b/docs/reference/document-loaders/requests.md new file mode 100644 index 00000000..11c2f6b6 --- /dev/null +++ b/docs/reference/document-loaders/requests.md @@ -0,0 +1,22 @@ +# :material-cloud-download: `RequestsDocumentLoader` + +`RequestsDocumentLoader` retrieves JSON-LD documents with `requests`. + +The default remote document loader uses `requests` when it is available. +Production applications should usually set at least a timeout: + +{{ example('document_loaders/requests_timeout.py', output_syntax='json') }} + +Use `secure=True` to require HTTPS URLs: + +{{ example('document_loaders/requests_secure.py', output_syntax='json') }} + +Extra keyword arguments are forwarded to `requests.get()`: + +{{ example('document_loaders/requests_extra_kwargs.py', output_syntax='json') }} + +Install the optional dependency with: + +```bash +pip install "PyLD[requests]" +``` diff --git a/docs/reference/index.md b/docs/reference/index.md new file mode 100644 index 00000000..bc376b99 --- /dev/null +++ b/docs/reference/index.md @@ -0,0 +1,21 @@ +--- +hide: [toc] +--- + +# :octicons-book-24: Reference + +
+ +- [:material-file-download-outline:{ .lg .middle } __Document Loaders__](document-loaders/) + + --- + + Load remote JSON-LD documents and contexts with the built-in loader classes. + +- :material-hard-hat:{ .lg .middle } __In construction__ + + --- + + We are working on more reference documentation. It is coming soon. + +
diff --git a/docs/requirements.txt b/docs/requirements.txt index 231419d2..f2391f32 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1,5 @@ -sphinx-autobuild +-r ../requirements.txt +typing_extensions +mkdocs-material==9.7.6 +mkdocs-macros-plugin==1.5.0 +mkdocs-awesome-pages-plugin==2.10.1 diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css new file mode 100644 index 00000000..562e6bbd --- /dev/null +++ b/docs/stylesheets/extra.css @@ -0,0 +1,51 @@ +.md-typeset .grid.cards.maintainers li { + display: grid; + grid-template-columns: auto 1fr; + grid-template-rows: auto auto; + align-items: center; + column-gap: 1rem; + row-gap: 0.25rem; +} + +.md-typeset .grid.cards.maintainers li > p:first-child { + grid-row: 1 / span 2; + grid-column: 1; + margin: 0; +} + +.md-typeset .grid.cards.maintainers li > p:first-child img { + width: 4rem; + height: 4rem; + border-radius: 50%; + display: block; +} + +.md-typeset .grid.cards.maintainers li > hr { + display: none; +} + +.md-typeset .grid.cards.maintainers li > p:nth-child(2), +.md-typeset .grid.cards.maintainers li > p:last-child { + grid-column: 2; + margin: 0; +} + +.md-typeset .grid.cards.maintainers li > p:nth-child(2) { + grid-row: 1; +} + +.md-typeset .grid.cards.maintainers li > p:last-child { + grid-row: 2; +} + +.md-typeset .admonition.example > .admonition-title { + display: flex; + justify-content: space-between; + align-items: center; + gap: 1rem; +} + +.md-typeset .admonition.example > .admonition-title .example-source-link { + margin-left: auto; + font-weight: normal; +} diff --git a/docs_macros.py b/docs_macros.py new file mode 100644 index 00000000..be94b563 --- /dev/null +++ b/docs_macros.py @@ -0,0 +1,138 @@ +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT_DIR = Path(__file__).resolve().parent +EXAMPLES_DIR = ROOT_DIR / 'docs' / 'examples' +sys.path.insert(0, str(ROOT_DIR / 'lib')) +sys.path.insert(0, str(ROOT_DIR / 'tests')) + +MANIFEST_BASES = { + 'frame-manifest': 'https://w3c.github.io/json-ld-framing/tests', + 'manifest-urgna2012': 'https://w3c.github.io/rdf-canon/tests', + 'manifest-urdna2015': 'https://w3c.github.io/rdf-canon/tests', +} +DEFAULT_TEST_BASE = 'https://w3c.github.io/json-ld-api/tests' + +_SKIP_ID_PATTERN = re.compile(r'^\.\*(?P[^#]+)#(?P[^$]+)\$$') + + +def _parse_skip_id_regex(pattern): + match = _SKIP_ID_PATTERN.fullmatch(pattern) + if not match: + return None + return match.group('manifest'), match.group('test_id') + + +def _test_url(manifest, test_id): + base = MANIFEST_BASES.get(manifest, DEFAULT_TEST_BASE) + return f'{base}/{manifest}#{test_id}' + + +def _example_path(name): + path = (EXAMPLES_DIR / name).resolve() + if not path.is_relative_to(EXAMPLES_DIR.resolve()): + raise ValueError(f'Invalid example path: {name}') + return path + + +def _github_branch(): + branch = os.environ.get('GITHUB_REF_NAME') + if branch: + return branch + result = subprocess.run( + ['git', 'symbolic-ref', '--short', 'HEAD'], + capture_output=True, + text=True, + cwd=ROOT_DIR, + ) + if result.returncode == 0 and result.stdout.strip(): + return result.stdout.strip() + return 'master' + + +def _example_github_url(name, repo_url): + rel_path = Path('docs/examples') / name + branch = 'master' + return f'{repo_url.rstrip("/")}/blob/{branch}/{rel_path.as_posix()}' + + +def define_env(env): + @env.macro + def bundled_contexts_table(): + from pyld import BUNDLED_CONTEXTS + + rows = [ + '| Context URL | Bundled file |', + '| --- | --- |', + ] + for url, path in sorted(BUNDLED_CONTEXTS.items()): + rows.append(f'| `{url}` | `{Path(path).name}` |') + return '\n'.join(rows) + + @env.macro + def skipped_tests_table(): + from runtests import TEST_TYPES + + rows = [ + '| Reason | Skipped tests |', + '| --- | --- |', + ] + + linked_tests = [] + seen_tests = set() + for test_type, config in sorted(TEST_TYPES.items()): + skip = config.get('skip', {}) + spec_versions = skip.get('specVersion', []) + if 'json-ld-1.0' in spec_versions: + rows.append( + f'| JSON-LD 1.0 processor behavior (`{test_type}`) | ' + f'All JSON-LD 1.0 tests |' + ) + + for pattern in skip.get('idRegex', []): + parsed = _parse_skip_id_regex(pattern) + if not parsed: + continue + manifest, test_id = parsed + url = _test_url(manifest, test_id) + if url in seen_tests: + continue + seen_tests.add(url) + linked_tests.append(f'[{test_id}]({url})') + + if linked_tests: + rows.append( + '| Explicitly skipped test cases | ' + ', '.join(linked_tests) + ' |' + ) + + return '\n'.join(rows) + + @env.macro + def example(name, output_syntax=None): + path = _example_path(name) + source = path.read_text() + result = subprocess.run( + [sys.executable, str(path)], + capture_output=True, + text=True, + check=True, + cwd=ROOT_DIR, + env={**os.environ, 'PYTHONPATH': str(ROOT_DIR / 'lib')}, + ) + github_url = _example_github_url(name, env.conf['repo_url']) + display_name = path.name + title = ( + f'Example' + f':fontawesome-brands-github: [`{display_name}`]({github_url})' + f'' + ) + output_lang = output_syntax or 'console' + body = ( + f'```python\n{source}```\n\n' + f'```{output_lang} title="Output"\n{result.stdout}```' + ) + indented = '\n'.join(f' {line}' for line in body.splitlines()) + return f'!!! example "{title}"\n\n{indented}\n' diff --git a/lib/pyld/context_resolver.py b/lib/pyld/context_resolver.py index 9b8d2c93..d479bb1f 100644 --- a/lib/pyld/context_resolver.py +++ b/lib/pyld/context_resolver.py @@ -24,7 +24,9 @@ class ContextResolver: Resolves and caches remote contexts. """ - def __init__(self, shared_cache, document_loader, max_context_urls: int = MAX_CONTEXT_URLS): + def __init__( + self, shared_cache, document_loader, max_context_urls: int = MAX_CONTEXT_URLS + ): """ Creates a ContextResolver. diff --git a/lib/pyld/jsonld.py b/lib/pyld/jsonld.py index a60fcd36..e603d001 100644 --- a/lib/pyld/jsonld.py +++ b/lib/pyld/jsonld.py @@ -1280,7 +1280,11 @@ def get_values(subject, property): :return: all of the values for a subject's property as an array. """ - return JsonLdProcessor.arrayify(subject.get(property)) if property in subject else [] + return ( + JsonLdProcessor.arrayify(subject.get(property)) + if property in subject + else [] + ) @staticmethod def remove_property(subject, property): @@ -1515,7 +1519,11 @@ def _compact(self, active_ctx, active_property, element, options): rval = self._compact_value( active_ctx, active_property, element, options ) - if isinstance(options['link'], dict) and options['link'] and _is_subject_reference(element): + if ( + isinstance(options['link'], dict) + and options['link'] + and _is_subject_reference(element) + ): # store linked element options['link'].setdefault(element['@id'], []).append( {'expanded': element, 'compacted': rval} @@ -1960,9 +1968,13 @@ def _compact(self, active_ctx, active_property, element, options): type_key = self._compact_iri(active_ctx, '@type') # Only object items can carry an embedded @type, # so only call .pop() on object to avoid AttributeError. - types = JsonLdProcessor.arrayify( - compacted_item.pop(type_key, []) - ) if _is_object(compacted_item) else [] + types = ( + JsonLdProcessor.arrayify( + compacted_item.pop(type_key, []) + ) + if _is_object(compacted_item) + else [] + ) key = types.pop(0) if types else None if types: JsonLdProcessor.add_value( @@ -2143,8 +2155,8 @@ def _expand( ) # prepare type-scoped contexts when nested - active_ctx, type_key, type_scoped_ctx = ( - self._prepare_nested_context(active_ctx, element, options) + active_ctx, type_key, type_scoped_ctx = self._prepare_nested_context( + active_ctx, element, options ) # process each key and value in element, ignoring @nest content @@ -2766,8 +2778,8 @@ def _expand_object( ) # prepare type-scoped contexts when nested - active_ctx, type_key, type_scoped_ctx = ( - self._prepare_nested_context(term_ctx, nv, options) + active_ctx, type_key, type_scoped_ctx = self._prepare_nested_context( + term_ctx, nv, options ) if [ @@ -4700,9 +4712,7 @@ def _validate_frame(self, frame): # @type must be wildcard, @json, or IRI if not ( _is_object(type_) or type_ == '@json' or _is_absolute_iri(type_) - ) or ( - _is_string(type_) and type_.startswith('_:') - ): + ) or (_is_string(type_) and type_.startswith('_:')): raise JsonLdError( 'Invalid JSON-LD syntax; invalid @type in frame.', 'jsonld.SyntaxError', diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..4786823d --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,41 @@ +site_name: PyLD +site_description: Python implementation of the JSON-LD API +site_url: https://digitalbazaar.github.io/pyld/ +repo_url: https://github.com/digitalbazaar/pyld +repo_name: digitalbazaar/pyld + +extra_css: + - stylesheets/extra.css + +theme: + name: material + features: + - content.code.copy + - navigation.footer + - navigation.indexes + - navigation.sections + - navigation.tabs + - navigation.top + - search.highlight + - search.suggest + +markdown_extensions: + - admonition + - attr_list + - md_in_html + - pymdownx.details + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.superfences + - toc: + permalink: true + +plugins: + - awesome-pages: + collapse_single_pages: true + - macros: + module_name: docs_macros + - search diff --git a/setup.py b/setup.py index 41fe18e9..62f533b1 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ os.path.dirname(__file__), 'lib', 'pyld', '__about__.py')) as fp: exec(fp.read(), about) -with open('README.rst') as fp: +with open('README.md') as fp: long_description = fp.read() setup( @@ -26,7 +26,7 @@ version=about['__version__'], description='Python implementation of the JSON-LD API', long_description=long_description, - long_description_content_type="text/x-rst", + long_description_content_type="text/markdown", author='Digital Bazaar', author_email='support@digitalbazaar.com', url='https://github.com/digitalbazaar/pyld', diff --git a/specifications/json-ld-api b/specifications/json-ld-api index 265e6b43..04a4eb7d 160000 --- a/specifications/json-ld-api +++ b/specifications/json-ld-api @@ -1 +1 @@ -Subproject commit 265e6b433a4eb25bad99d941c95f2ccecd6a8c1d +Subproject commit 04a4eb7dc7cbc313f3f5be7ad9a3b06e87741693 diff --git a/specifications/json-ld-framing b/specifications/json-ld-framing index 5437dbc5..fa228743 160000 --- a/specifications/json-ld-framing +++ b/specifications/json-ld-framing @@ -1 +1 @@ -Subproject commit 5437dbc5c0db543ccfa1b6b36197cc3687fc1b34 +Subproject commit fa228743e890499c35bc61aabf01e44cf5bbc3bc diff --git a/tests/test_jsonld.py b/tests/test_jsonld.py index f4fc22db..c602f18c 100644 --- a/tests/test_jsonld.py +++ b/tests/test_jsonld.py @@ -210,7 +210,6 @@ def test_base_does_not_expand_property_terms(self): } ] - def _make_context(self, num_terms): """Build a context with `num_terms` @type:@vocab terms sharing a scoped context.""" ctx = {"ex": "https://example.org/"} @@ -236,7 +235,6 @@ def _make_context(self, num_terms): ctx[f"prop{i}"] = f"ex:prop{i}" return ctx - def test_single_vocab_term_expands_correctly(self): """Single @type:@vocab term should expand bare string to @id.""" ctx = { @@ -249,8 +247,9 @@ def test_single_vocab_term_expands_correctly(self): } doc = {"@context": ctx, "Color": "Red"} result = jsonld.expand(doc) - assert result[0]["https://example.org/Color"] == [{"@id": "https://example.org/Red"}] - + assert result[0]["https://example.org/Color"] == [ + {"@id": "https://example.org/Red"} + ] def test_many_shared_scoped_contexts_expand_correctly(self): """ @@ -284,7 +283,6 @@ def test_many_shared_scoped_contexts_expand_correctly(self): f"EnumProp{i} did not expand to @id" ) - def test_last_vocab_term_expands_with_large_context(self): """The LAST @type:@vocab term in a large context must also expand correctly. @@ -295,8 +293,9 @@ def test_last_vocab_term_expands_with_large_context(self): # Only test the last term doc = {"@context": ctx, "EnumProp26": "TestValue"} result = jsonld.expand(doc) - assert result[0]["https://example.org/EnumProp26"] == [{"@id": "https://example.org/TestValue"}] - + assert result[0]["https://example.org/EnumProp26"] == [ + {"@id": "https://example.org/TestValue"} + ] def test_structured_value_still_works_with_scoped_context(self): """Structured values (objects) should still use the scoped context mappings.""" @@ -395,7 +394,6 @@ def test_scoped_context_on_nest_term_expands_nested_type_scoped_context(self): assert result == expected - def test_mixed_plain_and_vocab_terms(self): """Contexts with both plain and @type:@vocab terms should work correctly.""" ctx = { @@ -424,8 +422,12 @@ def test_mixed_plain_and_vocab_terms(self): } result = jsonld.expand(doc) expanded = result[0] - assert expanded["https://example.org/Color"] == [{"@id": "https://example.org/Blue"}] - assert expanded["https://example.org/Shape"] == [{"@id": "https://example.org/Circle"}] + assert expanded["https://example.org/Color"] == [ + {"@id": "https://example.org/Blue"} + ] + assert expanded["https://example.org/Shape"] == [ + {"@id": "https://example.org/Circle"} + ] assert expanded["https://example.org/name"] == [{"@value": "test"}] # Issue 145 @@ -470,6 +472,7 @@ def test_context_contained_with_propagate(self): expanded = jsonld.expand(input) assert expanded == expected + class TestFrame: # Issue 11 - PR: https://github.com/digitalbazaar/pyld/issues/149 """ @@ -794,36 +797,32 @@ def test_circular_references_link_and_embed(self): "@context": "http://schema.org", "@graph": [ { - "id": "http://www.janedoe.com", - "type": "Person", - "jobTitle": "Professor", - "knows": { - "id": "http://www.johnsmith.me", + "id": "http://www.janedoe.com", "type": "Person", + "jobTitle": "Professor", "knows": { - "id": "http://www.janedoe.com" + "id": "http://www.johnsmith.me", + "type": "Person", + "knows": {"id": "http://www.janedoe.com"}, + "name": "John Smith", }, - "name": "John Smith" - }, - "name": "Jane Doe", - "telephone": "(425) 123-4567" + "name": "Jane Doe", + "telephone": "(425) 123-4567", }, { - "id": "http://www.johnsmith.me", - "type": "Person", - "knows": { - "id": "http://www.janedoe.com", + "id": "http://www.johnsmith.me", "type": "Person", - "jobTitle": "Professor", "knows": { - "id": "http://www.johnsmith.me" + "id": "http://www.janedoe.com", + "type": "Person", + "jobTitle": "Professor", + "knows": {"id": "http://www.johnsmith.me"}, + "name": "Jane Doe", + "telephone": "(425) 123-4567", }, - "name": "Jane Doe", - "telephone": "(425) 123-4567" + "name": "John Smith", }, - "name": "John Smith" - } - ] + ], } frame = {'@context': 'http://schema.org', '@embed': '@once'} @@ -928,7 +927,7 @@ def test_compound_literal_direction_with_language(self): assert nquads == expected - # Issue 204 + # Issue 204 def test_conflicting_property_names(self): """ Conversion to RDF should allow a node in the root @context with @@ -955,7 +954,6 @@ def test_conflicting_property_names(self): nquads = jsonld.to_rdf(input, options={'format': 'application/n-quads'}) assert nquads == expected - def test_conflicting_property_names_in_nested_node(self): """ Conversion to RDF should not ignore a @nest'ed node in the root @context @@ -981,6 +979,7 @@ def test_conflicting_property_names_in_nested_node(self): nquads = jsonld.to_rdf(input, options={'format': 'application/n-quads'}) assert nquads == expected + class TestFromRDF: def test_compound_literal_direction_without_language(self): """ @@ -1117,6 +1116,7 @@ def test_compound_literal_invalid_language_fails(self): assert exc.value.code == 'invalid language-tagged string' + class TestCompact: # Issue 59 - PR: https://github.com/digitalbazaar/pyld/pull/60 def test_compaction_with_and_without_explicit_datatypes(self): @@ -1370,7 +1370,9 @@ def test_no_initial_context_drops_property(self): assert compacted == expected @pytest.mark.xfail - def test_no_initial_context_and_with_skip_expand_does_not_drop_property_whe_not_array(self): + def test_no_initial_context_and_with_skip_expand_does_not_drop_property_whe_not_array( + self, + ): """ Compacting document with singular value and without initial context should output the original input when skipExpansion is enabled. @@ -1378,11 +1380,15 @@ def test_no_initial_context_and_with_skip_expand_does_not_drop_property_whe_not_ input = {'name': 'Bob'} - compacted = jsonld.compact(input, {"@vocab": "http://example.org#"}, {"skipExpansion": True}) + compacted = jsonld.compact( + input, {"@vocab": "http://example.org#"}, {"skipExpansion": True} + ) expected = {"@context": {"@vocab": "http://example.org#"}, "name": "Bob"} assert compacted == expected - def test_no_initial_context_and_with_skip_expand_does_not_drop_property_when_array(self): + def test_no_initial_context_and_with_skip_expand_does_not_drop_property_when_array( + self, + ): """ Compacting document with array value and without initial context should output the original input when skipExpansion is enabled. @@ -1390,7 +1396,9 @@ def test_no_initial_context_and_with_skip_expand_does_not_drop_property_when_arr input = {'name': ['Bob']} - compacted = jsonld.compact(input, {"@vocab": "http://example.org#"}, {"skipExpansion": True}) + compacted = jsonld.compact( + input, {"@vocab": "http://example.org#"}, {"skipExpansion": True} + ) expected = {"@context": {"@vocab": "http://example.org#"}, "name": "Bob"} assert compacted == expected