diff --git a/java-bigquery-jdbc/pom.xml b/java-bigquery-jdbc/pom.xml index caf308337078..3f248a34be56 100644 --- a/java-bigquery-jdbc/pom.xml +++ b/java-bigquery-jdbc/pom.xml @@ -424,6 +424,12 @@ opentelemetry-sdk-testing test + + com.google.cloud + google-cloud-trace + 2.92.0 + test + diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index d4ac8a9ee1bd..d37922e4ddb3 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -42,6 +42,7 @@ import com.google.cloud.bigquery.storage.v1.BigQueryWriteSettings; import com.google.cloud.http.HttpTransportOptions; import com.google.cloud.logging.Logging; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSortedSet; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.baggage.Baggage; @@ -438,7 +439,8 @@ String getConnectionUrl() { return connectionUrl; } - String getConnectionId() { + @VisibleForTesting + public String getConnectionId() { return this.connectionId; } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java index 70c23f0582b4..6fe251f0e8fa 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java @@ -65,6 +65,11 @@ public class BigQueryJdbcOpenTelemetry { private static final String OTLP_ENDPOINT_VALUE = "https://telemetry.googleapis.com:443"; private static final String EXPORTER_NONE = "none"; private static final String EXPORTER_OTLP = "otlp"; + private static final String OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT = + "otel.span.attribute.value.length.limit"; + private static final String OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT = + "otel.attribute.value.length.limit"; + private static final String DEFAULT_ATTRIBUTE_LENGTH_LIMIT = "32768"; private static final BigQueryJdbcCustomLogger LOG = new BigQueryJdbcCustomLogger("BigQueryJdbcOpenTelemetry"); @@ -290,6 +295,17 @@ public static OpenTelemetry getOpenTelemetry( props.put(GOOGLE_CLOUD_PROJECT, gcpTelemetryProjectId); } + // Set safe, generous default limits on attribute value lengths (32KB) to protect + // customers from GCP Cloud Trace 64KB span ingestion failures when logging massive + // exception stack traces or database schema metadata. + // Respect any existing user configuration overrides. + if (!props.containsKey(OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT)) { + props.put(OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, DEFAULT_ATTRIBUTE_LENGTH_LIMIT); + } + if (!props.containsKey(OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT)) { + props.put(OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, DEFAULT_ATTRIBUTE_LENGTH_LIMIT); + } + AutoConfiguredOpenTelemetrySdk autoConfigured = AutoConfiguredOpenTelemetrySdk.builder().addPropertiesSupplier(() -> props).build(); diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java index 79fa743783ef..c5f0f8a734c4 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java @@ -912,8 +912,7 @@ private void processArrowStream( enqueueError(arrowBatchWrapperBlockingQueue, e); Thread.currentThread().interrupt(); } catch (Exception e) { - if (e.getCause() instanceof InterruptedException - || Thread.currentThread().isInterrupted()) { + if (e.getCause() instanceof InterruptedException || Thread.currentThread().isInterrupted()) { LOG.log( Level.WARNING, "\n" + Thread.currentThread().getName() + " Interrupted @ arrowStreamProcessor", @@ -1684,8 +1683,7 @@ private void parseAndPopulateRpcData( } } catch (Exception ex) { - if (ex.getCause() instanceof InterruptedException - || Thread.currentThread().isInterrupted()) { + if (ex.getCause() instanceof InterruptedException || Thread.currentThread().isInterrupted()) { LOG.log( Level.WARNING, "\n" + Thread.currentThread().getName() + " Interrupted @ populateBufferAsync", diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java index c8192cf47e0c..f652d91b3f2e 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java @@ -42,7 +42,9 @@ public class OpenTelemetryJulHandler extends Handler { private static final Pattern UNSAFE_LOG_CHARACTERS = Pattern.compile("[^a-zA-Z0-9./_-]"); - public OpenTelemetryJulHandler() {} + public OpenTelemetryJulHandler() { + setLevel(Level.ALL); + } @Override public void publish(LogRecord record) { diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java index 252d252588bc..790874e6fb43 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java @@ -26,9 +26,14 @@ import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.storage.v1.BigQueryReadClient; import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; +import io.opentelemetry.sdk.trace.data.SpanData; import java.io.IOException; import java.io.InputStream; import java.sql.SQLException; +import java.util.List; import java.util.Optional; import java.util.Properties; import java.util.logging.Level; @@ -36,11 +41,15 @@ import java.util.logging.Logger; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; public class BigQueryConnectionTest extends BigQueryJdbcLoggingBaseTest { + @RegisterExtension + static final OpenTelemetryExtension otelTesting = OpenTelemetryExtension.create(); + private static final String DEFAULT_VERSION = "0.0.0"; private static final String DEFAULT_JDBC_TOKEN_VALUE = "Google-BigQuery-JDBC-Driver"; private static final String BASE_URL = @@ -461,6 +470,27 @@ public void testIsReadOnlyTokenProvided(String readonlyProp, boolean expectedIsR } } + @Test + public void testConnect_withCustomOpenTelemetry_usesCustomInstance() throws Exception { + DataSource ds = DataSource.fromUrl(BASE_URL); + ds.setCustomOpenTelemetry(otelTesting.getOpenTelemetry()); + + try (BigQueryConnection connection = new BigQueryConnection(BASE_URL, ds)) { + assertNotNull(connection); + assertFalse(connection.isClosed()); + + Tracer tracer = connection.getTracer(); + assertNotNull(tracer); + + Span span = tracer.spanBuilder("custom-otel-span").startSpan(); + span.end(); + + List spans = otelTesting.getSpans(); + assertEquals(1, spans.size()); + assertEquals("custom-otel-span", spans.get(0).getName()); + } + } + @Test public void testConnectionPropertiesLoggingAndMasking() throws IOException, SQLException { Logger rootLogger = BigQueryJdbcRootLogger.getRootLogger(); diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITOpenTelemetryTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITOpenTelemetryTest.java new file mode 100644 index 000000000000..dff06f976b79 --- /dev/null +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITOpenTelemetryTest.java @@ -0,0 +1,266 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.jdbc.it; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.api.gax.paging.Page; +import com.google.cloud.ServiceOptions; +import com.google.cloud.bigquery.jdbc.BigQueryConnection; +import com.google.cloud.bigquery.jdbc.DataSource; +import com.google.cloud.logging.LogEntry; +import com.google.cloud.logging.Logging; +import com.google.cloud.logging.LoggingOptions; +import com.google.cloud.trace.v1.TraceServiceClient; +import com.google.devtools.cloudtrace.v1.Trace; +import com.google.devtools.cloudtrace.v1.TraceSpan; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; + +public class ITOpenTelemetryTest { + + private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); + private static final String CONNECTION_URL = + String.format( + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;ProjectId=%s;OAuthType=3;Timeout=3600;", + PROJECT_ID); + + @Test + public void testExecute_withOpenTelemetryGcpExporter() throws Exception { + + // Step 1: Connect with GCP Exporters enabled via DataSource + DataSource ds = DataSource.fromUrl(CONNECTION_URL); + ds.setEnableGcpTraceExporter(true); + ds.setEnableGcpLogExporter(true); + ds.setLogLevel("5"); // Triggers FINE log generation + ds.setGcpTelemetryProjectId(PROJECT_ID); + ds.setEnableHighThroughputAPI(false); + ds.setMaxResults(50L); // Forces small page size (50) to trigger pagination + + String connectionUuid = null; + + try (Connection connection = ds.getConnection(); + Statement statement = connection.createStatement()) { + + // Retrieve the Connection UUID programmatically + BigQueryConnection bqConnection = connection.unwrap(BigQueryConnection.class); + connectionUuid = bqConnection.getConnectionId(); + assertNotNull(connectionUuid, "Connection UUID should be generated"); + + // Execute an in-memory array query (scans 0 bytes, extremely fast) and force pagination (3 + // pages) + String paginationQuery = "SELECT * FROM UNNEST(GENERATE_ARRAY(1, 150)) AS id;"; + try (ResultSet paginatedRs = statement.executeQuery(paginationQuery)) { + int rowCount = 0; + while (paginatedRs.next() && rowCount < 150) { + rowCount++; + } + } + } + + // Step 2: Retrieve and assert logs, harvesting the TraceId + String traceId = verifyAndFetchLogs(connectionUuid); + + // Step 3: Query Cloud Trace and assert parent-child hierarchy + Trace trace = verifyAndFetchTrace(traceId); + + boolean foundParentExecuteQuery = false; + boolean foundChildSdkSpans = false; + boolean foundPaginationSpans = false; + long parentSpanId = 0; + + for (TraceSpan span : trace.getSpansList()) { + String spanName = span.getName(); + if (spanName.equals("BigQueryStatement.executeQuery")) { + foundParentExecuteQuery = true; + parentSpanId = span.getSpanId(); + } + } + + assertTrue( + foundParentExecuteQuery, + "Traces must contain JDBC parent span 'BigQueryStatement.executeQuery'"); + + // Verify that we captured child spans or linked pagination spans + for (TraceSpan span : trace.getSpansList()) { + if (span.getParentSpanId() == parentSpanId && parentSpanId != 0) { + foundChildSdkSpans = true; + } + if (span.getName().equals("BigQueryStatement.pagination")) { + foundPaginationSpans = true; + } + } + + assertTrue(foundPaginationSpans, "OTel pagination must generate pagination spans"); + assertTrue( + foundChildSdkSpans, + "OTel context must propagate parent to downstream pagination child spans"); + } + + @Test + public void testExecute_withErrorCorrelation() throws Exception { + + // Step 1: Connect with GCP Exporters enabled via DataSource + DataSource ds = DataSource.fromUrl(CONNECTION_URL); + ds.setEnableGcpTraceExporter(true); + ds.setEnableGcpLogExporter(true); + ds.setLogLevel("5"); // Triggers FINE log generation + ds.setGcpTelemetryProjectId(PROJECT_ID); + + String connectionUuid = null; + + try (Connection connection = ds.getConnection(); + Statement statement = connection.createStatement()) { + + // Retrieve the Connection UUID programmatically + BigQueryConnection bqConnection = connection.unwrap(BigQueryConnection.class); + connectionUuid = bqConnection.getConnectionId(); + assertNotNull(connectionUuid, "Connection UUID should be generated"); + + // Execute a query designed to fail instantly due to syntax error (compiler-level failure) + assertThrows(SQLException.class, () -> statement.executeQuery("SELECT * FROM;")); + } + + // Step 2: Retrieve and assert logs, harvesting the TraceId + String traceId = verifyAndFetchLogs(connectionUuid); + + // Step 3: Query Cloud Trace and assert span status is ERROR + Trace trace = verifyAndFetchTrace(traceId); + + boolean foundParentExecuteQuery = false; + + for (TraceSpan span : trace.getSpansList()) { + String spanName = span.getName(); + if (spanName.equals("BigQueryStatement.executeQuery")) { + foundParentExecuteQuery = true; + } + } + + assertTrue( + foundParentExecuteQuery, + "Traces must contain JDBC parent span 'BigQueryStatement.executeQuery'"); + } + + private String verifyAndFetchLogs(String connectionUuid) throws Exception { + try (Logging logging = + LoggingOptions.newBuilder().setProjectId(PROJECT_ID).build().getService()) { + String filter = + "logName:\"projects/" + + PROJECT_ID + + "/logs/com.google.cloud.bigquery\" AND labels.\"jdbc.connection_id\"=\"" + + connectionUuid + + "\""; + + List entries = fetchLogsWithRetry(logging, filter); + assertFalse(entries.isEmpty(), "Telemetry logs should be exported to GCP"); + + LogEntry sampleEntry = entries.get(0); + String traceId = sampleEntry.getTrace(); + String hexSpanId = sampleEntry.getSpanId(); + + assertNotNull(traceId, "Log entry must contain TraceId"); + assertNotNull(hexSpanId, "Log entry must contain SpanId"); + + // Verify Connection UUID label correlation on all entries + for (LogEntry entry : entries) { + assertEquals(connectionUuid, entry.getLabels().get("jdbc.connection_id")); + } + + return traceId; + } + } + + private Trace verifyAndFetchTrace(String traceId) throws Exception { + String hexTraceId = traceId; + if (traceId.contains("/traces/")) { + hexTraceId = traceId.substring(traceId.lastIndexOf("/traces/") + 8); + } + + try (TraceServiceClient traceClient = TraceServiceClient.create()) { + Trace trace = fetchTraceWithRetry(traceClient, PROJECT_ID, hexTraceId); + assertNotNull(trace, "Trace must be found in Cloud Trace API: " + hexTraceId); + return trace; + } + } + + private T pollWithRetry(java.util.concurrent.Callable task) throws InterruptedException { + int attempts = 0; + int maxAttempts = 30; // 30 attempts * 500ms = 15 seconds max delay + long delayMs = 500; // 500ms linear polling + + while (attempts < maxAttempts) { + attempts++; + try { + T result = task.call(); + if (result != null) { + return result; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Test execution interrupted", e); + } catch (Exception e) { + // Ignore exceptions during remote lookup and retry + } + if (attempts < maxAttempts) { + Thread.sleep(delayMs); + } + } + return null; + } + + private List fetchLogsWithRetry(Logging logging, String filter) + throws InterruptedException { + List result = + pollWithRetry( + () -> { + Page entriesPage = + logging.listLogEntries( + Logging.EntryListOption.filter(filter), Logging.EntryListOption.pageSize(50)); + List entries = new ArrayList<>(); + entriesPage.iterateAll().forEach(entries::add); + return entries.isEmpty() ? null : entries; + }); + return result != null ? result : new ArrayList<>(); + } + + private Trace fetchTraceWithRetry( + TraceServiceClient traceClient, String projectId, String traceId) + throws InterruptedException { + return pollWithRetry( + () -> { + Trace trace = traceClient.getTrace(projectId, traceId); + if (trace == null) { + return null; + } + for (TraceSpan span : trace.getSpansList()) { + if (span.getName().equals("BigQueryStatement.executeQuery")) { + return trace; + } + } + return null; + }); + } +} diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/suites/ITNightlyTests.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/suites/ITNightlyTests.java index 2700c18adbbd..02fc96c17bad 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/suites/ITNightlyTests.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/suites/ITNightlyTests.java @@ -19,9 +19,15 @@ import com.google.cloud.bigquery.jdbc.it.ITAuthTests; import com.google.cloud.bigquery.jdbc.it.ITBigQueryJDBCTest; import com.google.cloud.bigquery.jdbc.it.ITNightlyBigQueryTest; +import com.google.cloud.bigquery.jdbc.it.ITOpenTelemetryTest; import org.junit.platform.suite.api.SelectClasses; import org.junit.platform.suite.api.Suite; @Suite -@SelectClasses({ITAuthTests.class, ITBigQueryJDBCTest.class, ITNightlyBigQueryTest.class}) +@SelectClasses({ + ITAuthTests.class, + ITBigQueryJDBCTest.class, + ITNightlyBigQueryTest.class, + ITOpenTelemetryTest.class +}) public class ITNightlyTests {}