docs(vrl): Automatically generate VRL function documentation

.github/actions/setup/action.yml

@@ -211,7 +211,7 @@ runs:
         EOF
 
     - name: Install protoc
-      if: ${{ inputs.protoc == 'true' }}
+      if: ${{ inputs.protoc == 'true' || env.VDEV_NEEDS_COMPILE == 'true' }}
       shell: bash
       run: |
         echo "Installing protoc"

.github/actions/spelling/expect.txt

@@ -677,3 +677,28 @@ zork
 zorp
 zsherman
 zulip
+QAAAA
+LAAAAPAd
+VGhl
+FAJs
+VISU
+ISU
+Ayv
+Bym
+MGkzw
+TLp
+LOk
+vsiz
+Bpd
+Glu
+Nvb
+ZWVs
+CBm
+Fyd
+Uge
+Nhbid
+Qgb
+IGRI
+Vzc
+Ugc
+IGhhdm

.github/workflows/changes.yml

@@ -222,7 +222,8 @@ jobs:
           component_docs:
             - 'scripts/generate-component-docs.rb'
             - "vdev/**"
-            - 'website/cue/**/base/**.cue'
+            - 'website/cue/**/generated/**.cue'
+            - 'website/cue/**/functions/*.cue'
             - ".github/workflows/changes.yml"
           markdown:
             - '**/**.md'

.github/workflows/test.yml

@@ -146,8 +146,8 @@ jobs:
           markdownlint: true
       - run: make check-markdown
 
-  check-component-docs:
-    name: Check Component Docs
+  check-generated-docs:
+    name: Check Generated Docs
     runs-on: ubuntu-24.04-8core
     if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.component_docs == 'true' || needs.changes.outputs.test-yml == 'true' }}
     needs: changes
@@ -159,7 +159,7 @@ jobs:
           protoc: true
           cue: true
           libsasl2: true
-      - run: make check-component-docs
+      - run: make check-generated-docs
 
   check-rust-docs:
     name: Check Rust Docs
@@ -217,7 +217,7 @@ jobs:
       - check-licenses
       - check-docs
       - check-markdown
-      - check-component-docs
+      - check-generated-docs
       - check-rust-docs
       - test-vrl
       - build-vrl-playground

CONTRIBUTING.md

@@ -141,7 +141,7 @@ echo "Running pre-push checks..."
 make check-licenses
 make check-fmt
 make check-clippy
-make check-component-docs
+make check-generated-docs
 
 # Some other checks that in our experience rarely fail on PRs.
 make check-deny
@@ -292,7 +292,7 @@ cargo vdev check events
 cargo vdev check licenses
 # Vector's documentation for each component is generated from the comments attached to the Component structs and members.
 # Running this ensures that the generated docs are up to date.
-make check-component-docs
+make check-generated-docs
 # Generate the code documentation for the Vector project.
 # Run this to ensure the docs can be generated without errors (warnings are acceptable at the minute).
 cd rust-doc && make docs

Makefile

@@ -472,7 +472,7 @@ check: ## Run prerequisite code checks
 check-all: ## Check everything
 check-all: check-fmt check-clippy check-docs
 check-all: check-examples check-component-features
-check-all: check-scripts check-deny check-component-docs check-licenses
+check-all: check-scripts check-deny check-generated-docs check-licenses
 
 .PHONY: check-component-features
 check-component-features: ## Check that all component features are setup properly
@@ -514,9 +514,9 @@ check-deny: ## Check advisories licenses and sources for crate dependencies
 check-events: ## Check that events satisfy patterns set in https://github.com/vectordotdev/vector/blob/master/rfcs/2020-03-17-2064-event-driven-observability.md
 	${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check events
 
-.PHONY: check-component-docs
-check-component-docs: generate-component-docs ## Checks that the machine-generated component Cue docs are up-to-date.
-	${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check component-docs
+.PHONY: check-generated-docs
+check-generated-docs: generate-docs ## Checks that the machine-generated component Cue docs are up-to-date.
+	${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check generated-docs
 
 ##@ Rustdoc
 build-rustdoc: ## Build Vector's Rustdocs
@@ -693,6 +693,14 @@ generate-component-docs: ## Generate per-component Cue docs from the configurati
 	${MAYBE_ENVIRONMENT_EXEC} $(VDEV) build component-docs /tmp/vector-config-schema.json \
 		$(if $(findstring true,$(CI)),>/dev/null,)
 
+.PHONY: generate-vrl-docs
+generate-vrl-docs: ## Generate VRL function documentation from Rust source.
+	${MAYBE_ENVIRONMENT_EXEC} $(VDEV) build vrl-docs \
+		$(if $(findstring true,$(CI)),>/dev/null,)
+
+.PHONY: generate-docs
+generate-docs: generate-component-docs generate-vrl-docs
+
 .PHONY: signoff
 signoff: ## Signsoff all previous commits since branch creation
 	scripts/signoff.sh

docs/DEVELOPING.md

@@ -127,7 +127,7 @@ Loosely, you'll need the following:
 - **To run integration tests:** Have `docker` available, or a real live version of that service. (Use `AUTOSPAWN=false`)
 - **To run `make check-component-features`:** Have `remarshal` installed.
 - **To run `make check-licenses` or `make build-licenses`:** Have `dd-rust-license-tool` [installed](https://github.com/DataDog/rust-license-tool).
-- **To run `make generate-component-docs`:** Have `cue` [installed](https://cuelang.org/docs/install/).
+- **To run `make generate-docs`:** Have `cue` [installed](https://cuelang.org/docs/install/).
 
 If you find yourself needing to run something inside the Docker environment described above, that's totally fine, they won't collide or hurt each other. In this case, you'd just run `make environment-generate`.
 
@@ -161,8 +161,8 @@ cargo bench transforms::example
 # Format your code before pushing!
 make fmt
 cargo fmt
-# Build component documentation for the website
-make generate-component-docs
+# Build component and VRL documentation for the website
+make generate-docs
 ```
 
 If you run `make` you'll see a full list of all our tasks. Some of these will start Docker containers, sign commits, or even make releases. These are not common development commands and your mileage may vary.

docs/DOCUMENTING.md

@@ -61,7 +61,7 @@ Much of Vector's reference documentation is automatically compiled from source c
 To regenerate this content, run:
 
 ```bash
-make generate-component-docs
+make generate-docs
 ```
 
 ### Formatting

scripts/cue.sh

@@ -24,7 +24,8 @@ cmd_list() {
 }
 
 cmd_fmt() {
-  list-docs-files | xargs cue fmt "$@"
+  # Ignore JSON-style cue files generated from VRL source code
+  list-docs-files | grep -v "${CUE_SOURCES}/reference/remap/functions/" | xargs cue fmt "$@"
 }
 
 cmd_vet() {

vdev/Cargo.toml

@@ -35,7 +35,7 @@ paste.workspace = true
 regex.workspace = true
 reqwest = { workspace = true, features = ["blocking"] }
 serde.workspace = true
-serde_json.workspace = true
+serde_json = { workspace = true, features = ["preserve_order"] }
 serde_yaml.workspace = true
 sha2 = "0.10.9"
 tempfile.workspace = true
@@ -45,6 +45,9 @@ semver.workspace = true
 indoc.workspace = true
 git2 = { version = "0.20.4" }
 cfg-if.workspace = true
+vector-vrl-functions = { path = "../lib/vector-vrl/functions", features = ["dnstap", "vrl-metrics"] }
+# Only here for docs generation. Using vrl with this feature enabled will be severely broken
+vrl = { workspace = true, features = ["__mock_return_values_for_tests"] }
 
 [package.metadata.binstall]
 pkg-url = "{ repo }/releases/download/vdev-v{ version }/{ name }-{ target }-v{ version }.tgz"

vdev/src/commands/build/mod.rs

@@ -1,6 +1,7 @@
 mod licenses;
 mod publish_metadata;
 mod vector;
+mod vrl_docs;
 mod vrl_wasm;
 
 crate::cli_subcommands! {
@@ -11,6 +12,7 @@ crate::cli_subcommands! {
     publish_metadata,
     release_cue,
     vector,
+    vrl_docs,
     vrl_wasm,
 }