Added new scenario and updated FIPS testing related documentation

.github/workflows/build.yaml

@@ -261,6 +261,15 @@ jobs:
           source ~/venv/qa/bin/activate
           export CLICKHOUSE_TESTS_DIR=$(pwd)/test/testflows/clickhouse_backup
           ./test/testflows/run.sh
+      - name: Report FIPS testflows coverage
+        uses: coverallsapp/github-action@v2
+        with:
+          fail-on-error: false
+          base-path: ./
+          file: test/testflows/_coverage_/coverage.out
+          parallel: true
+          format: golang
+          flag-name: testflows-${{ matrix.clickhouse }}
 
       - name: Fix FIPS log permissions for artifact upload
         if: always()
@@ -414,6 +423,7 @@ jobs:
     needs:
       - test
       - testflows
+      - testflows_fips
     name: coverage
     runs-on: ubuntu-24.04
     steps:

test/integration/containers.go

@@ -324,7 +324,7 @@ func (tc *TestContainers) RestartContainer(t *testing.T, name string) error {
 	if err := tc.client.ContainerRestart(ctx, info.ID, container.StopOptions{Timeout: &timeout}); err != nil {
 		return err
 	}
-	return tc.waitHealthy(ctx, name, 10*time.Minute, t.Name())
+	return tc.waitHealthy(ctx, name, 12*time.Minute, t.Name())
 }
 
 func (tc *TestContainers) waitHealthy(ctx context.Context, name string, timeout time.Duration, testName string) error {

test/testflows/clickhouse_backup/configs/clickhouse_nonfips_server/config.d/listeners-fips-cipher-stress.xml

@@ -0,0 +1,32 @@
+<!--
+    Stress variant of `listeners-fips-cipher.xml`, mounted on the
+    non-FIPS Altinity ClickHouse server by the stress arm of
+    `outbound_tls_to_nonfips_clickhouse_with_cipher_profile`
+    (enabled by the `stress` regression flag).
+
+    Instead of a single cipher, this offers the FULL FIPS-approved cipher
+    set documented for Altinity FIPS-compatible builds, so the end-to-end
+    `clickhouse-backup-fips tables` run is exercised against the same
+    cipher list a real FIPS-compatible server would advertise. See:
+    https://docs.altinity.com/altinitystablebuilds/fips-compatible-altinity-builds/
+
+    The plain `tcp_port` (9000) and `http_port` (8123) are left as image
+    defaults so the container's healthcheck (`/ping` on 8123) keeps working.
+
+    Server certificate / key paths point at `/etc/clickhouse-server/ssl/`
+    which the test bind-mounts from `configs/clickhouse/ssl/`.
+-->
+<clickhouse>
+    <tcp_port_secure>9440</tcp_port_secure>
+    <openSSL>
+        <server>
+            <certificateFile>/etc/clickhouse-server/ssl/server.crt</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/ssl/server.key</privateKeyFile>
+            <cipherList>ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384</cipherList>
+            <cipherSuites>TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384</cipherSuites>
+            <preferServerCiphers>true</preferServerCiphers>
+            <disableProtocols>sslv2,sslv3,tlsv1,tlsv1_1</disableProtocols>
+            <verificationMode>relaxed</verificationMode>
+        </server>
+    </openSSL>
+</clickhouse>

test/testflows/clickhouse_backup/regression.py

@@ -18,9 +18,7 @@
 
 from clickhouse_backup.requirements.requirements import *
 
-from clickhouse_backup.requirements.fips.requirements import (
-    QA_SRS013_ClickHouse_Backup_Utility_FIPS_Compatibility,
-)
+from clickhouse_backup.requirements.fips.requirements import *
 from clickhouse_backup.tests.common import simple_data_types_columns
 
 # `--fips-godebug` choices mapped to the `GODEBUG` value exported on
@@ -31,6 +29,7 @@
 #   * `only`  - FIPS active with strict enforcement (default).
 #   * `off`   - FIPS disabled at runtime.
 FIPS_GODEBUG_VALUES = {
+    "empty": "",
     "unset": None,
     "on": "fips140=on",
     "only": "fips140=only",
@@ -105,6 +104,9 @@ def regression(self, local, stress=False, fips=True, fips_godebug="only"):
             self.context.backup_config_origin = origin_path
             self.context.backup_config_file = config_path
             self.context.cluster = cluster
+            # `--stress` widens the FIPS cipher/suite coverage (see tests/fips_140_3.py).
+            # Default runs keep the documented minimum so they stay fast.
+            self.context.stress = stress
             self.context.nodes = [self.context.cluster.node(n) for n in ["clickhouse1", "clickhouse2"]]
             self.context.backup = self.context.cluster.node("clickhouse_backup")
             self.context.kafka = self.context.cluster.node("kafka")

test/testflows/clickhouse_backup/requirements/fips/requirements.md

@@ -18,9 +18,10 @@
         * 4.1.1 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.GoCryptographicModule](#rqsrs-013clickhousebackuputilityfipsgocryptographicmodule)
         * 4.1.2 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.Build.GOFIPS140](#rqsrs-013clickhousebackuputilityfipsbuildgofips140)
         * 4.1.3 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.Binary](#rqsrs-013clickhousebackuputilityfipsbinary)
-        * 4.1.4 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.Approved.TLSProtocolVersions](#rqsrs-013clickhousebackuputilityfipsapprovedtlsprotocolversions)
-        * 4.1.5 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.Approved.CipherSuites.TLSv12.Approved](#rqsrs-013clickhousebackuputilityfipsapprovedciphersuitestlsv12approved)
-        * 4.1.6 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.Approved.CipherSuites.TLSv13.Approved](#rqsrs-013clickhousebackuputilityfipsapprovedciphersuitestlsv13approved)
+        * 4.1.4 [RQ.SRS-013.ClickHouse.BackupUtility.non-FIPS.Binary](#rqsrs-013clickhousebackuputilitynon-fipsbinary)
+        * 4.1.5 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.Approved.TLSProtocolVersions](#rqsrs-013clickhousebackuputilityfipsapprovedtlsprotocolversions)
+        * 4.1.6 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.Approved.CipherSuites.TLSv12.Approved](#rqsrs-013clickhousebackuputilityfipsapprovedciphersuitestlsv12approved)
+        * 4.1.7 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.Approved.CipherSuites.TLSv13.Approved](#rqsrs-013clickhousebackuputilityfipsapprovedciphersuitestlsv13approved)
     * 4.2 [Connectivity](#connectivity)
         * 4.2.1 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.Connectivity.FIPSEndpoint](#rqsrs-013clickhousebackuputilityfipsconnectivityfipsendpoint)
         * 4.2.2 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.Connectivity.NonFIPSEndpoint](#rqsrs-013clickhousebackuputilityfipsconnectivitynonfipsendpoint)
@@ -29,8 +30,10 @@
         * 4.3.2 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.Version.BuildSetting](#rqsrs-013clickhousebackuputilityfipsversionbuildsetting)
     * 4.4 [GODEBUG fips140 Modes](#godebug-fips140-modes)
         * 4.4.1 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.GODEBUG.Unset](#rqsrs-013clickhousebackuputilityfipsgodebugunset)
-        * 4.4.2 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.GODEBUG.On](#rqsrs-013clickhousebackuputilityfipsgodebugon)
-        * 4.4.3 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.GODEBUG.Only](#rqsrs-013clickhousebackuputilityfipsgodebugonly)
+        * 4.4.2 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.GODEBUG.Empty](#rqsrs-013clickhousebackuputilityfipsgodebugempty)
+        * 4.4.3 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.GODEBUG.Off](#rqsrs-013clickhousebackuputilityfipsgodebugoff)
+        * 4.4.4 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.GODEBUG.On](#rqsrs-013clickhousebackuputilityfipsgodebugon)
+        * 4.4.5 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.GODEBUG.Only](#rqsrs-013clickhousebackuputilityfipsgodebugonly)
     * 4.5 [Startup Integrity Self-Tests](#startup-integrity-self-tests)
         * 4.5.1 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.SelfTest.Integrity](#rqsrs-013clickhousebackuputilityfipsselftestintegrity)
         * 4.5.2 [RQ.SRS-013.ClickHouse.BackupUtility.FIPS.SelfTest.TamperedBinary](#rqsrs-013clickhousebackuputilityfipsselftesttamperedbinary)
@@ -125,6 +128,12 @@ version: 1.0
 The FIPS-compatible build of the [clickhouse-backup] utility SHALL be distributed as a separate
 binary named `clickhouse-backup-fips`, distinct from the standard `clickhouse-backup` binary.
 
+#### RQ.SRS-013.ClickHouse.BackupUtility.non-FIPS.Binary
+version: 1.0
+
+The regular build of the [clickhouse-backup] utility SHALL be distributed as a
+binary named `clickhouse-backup`, that SHALL report ``FIPS 140-3:	false``
+
 #### RQ.SRS-013.ClickHouse.BackupUtility.FIPS.Approved.TLSProtocolVersions
 version: 1.0
 
@@ -190,29 +199,56 @@ The output of `go version -m $(which clickhouse-backup-fips)` SHALL contain the
 
 ### GODEBUG fips140 Modes
 
+The [clickhouse-backup-fips] binary SHALL expose its FIPS build and runtime posture via
+`clickhouse-backup-fips --fips-info`, which prints a line-oriented `key: value` dump including,
+under the `fips_module:` block, `enabled: <true|false>` and `enforced: <true|false>`. The binary
+is built with `DefaultGODEBUG=fips140=on`, so the `fips140` runtime key SHALL produce the
+following posture:
+
+| `GODEBUG` runtime | `enabled` | `enforced` |
+| ----------------- | --------- | ---------- |
+| unset             | true      | false      |
+| empty (`GODEBUG=`)| true      | false      |
+| `fips140=off`     | false     | false      |
+| `fips140=on`      | true      | false      |
+| `fips140=only`    | true      | true       |
+
 #### RQ.SRS-013.ClickHouse.BackupUtility.FIPS.GODEBUG.Unset
 version: 1.0
 
-When `GODEBUG` is not set, the [clickhouse-backup-fips] binary SHALL operate with FIPS 140-3 mode
-enabled by build-time default, `--version` SHALL report `FIPS 140-3: true`, and the basic
-`clickhouse-backup-fips tables` command SHALL return the list of tables from a FIPS-configured
-ClickHouse endpoint.
+When `GODEBUG` is not set, the [clickhouse-backup-fips] binary SHALL rely on its build-time default
+(`DefaultGODEBUG=fips140=on`) and operate with FIPS 140-3 mode enabled but not enforced. The
+output of `clickhouse-backup-fips --fips-info` SHALL report `enabled: true` and `enforced: false`.
+
+#### RQ.SRS-013.ClickHouse.BackupUtility.FIPS.GODEBUG.Empty
+version: 1.0
+
+When started with an empty `GODEBUG` (i.e. `GODEBUG=`), the [clickhouse-backup-fips] binary SHALL
+behave identically to the unset case, relying on its build-time default (`fips140=on`) with
+FIPS 140-3 mode enabled but not enforced. The output of `clickhouse-backup-fips --fips-info`
+SHALL report `enabled: true` and `enforced: false`.
+
+#### RQ.SRS-013.ClickHouse.BackupUtility.FIPS.GODEBUG.Off
+version: 1.0
+
+When started with `GODEBUG=fips140=off`, the [clickhouse-backup-fips] binary SHALL disable
+FIPS 140-3 mode. The output of `clickhouse-backup-fips --fips-info` SHALL report `enabled: false`
+and `enforced: false`.
 
 #### RQ.SRS-013.ClickHouse.BackupUtility.FIPS.GODEBUG.On
 version: 1.0
 
 When started with `GODEBUG=fips140=on`, the [clickhouse-backup-fips] binary SHALL operate with
-FIPS 140-3 mode enabled without strict enforcement, `--version` SHALL report `FIPS 140-3: true`,
-and the basic `clickhouse-backup-fips tables` command SHALL return the list of tables from a
-FIPS-configured ClickHouse endpoint.
+FIPS 140-3 mode enabled without strict enforcement. The output of
+`clickhouse-backup-fips --fips-info` SHALL report `enabled: true` and `enforced: false`.
 
 #### RQ.SRS-013.ClickHouse.BackupUtility.FIPS.GODEBUG.Only
 version: 1.0
 
 When started with `GODEBUG=fips140=only`, the [clickhouse-backup-fips] binary SHALL operate with
 strict FIPS 140-3 enforcement so that any non-approved cryptographic operation triggers an error
-or panic, `--version` SHALL report `FIPS 140-3: true`, and `clickhouse-backup-fips tables` against
-an approved [TLS] configuration SHALL return the list of tables.
+or panic. The output of `clickhouse-backup-fips --fips-info` SHALL report `enabled: true` and
+`enforced: true`.
 
 ### Startup Integrity Self-Tests