From 59867d3f319fc6b623e54038ba58eb190f6256e5 Mon Sep 17 00:00:00 2001 From: Vishal Kumar Singh Date: Tue, 21 Apr 2026 16:57:23 +0000 Subject: [PATCH] fix(state): trim noisy WARN in RemovedPartitionState to topic-partition+size+epoch When many partitions are reassigned away from a consumer, the WARN log in `RemovedPartitionState#maybeRegisterNewPollBatchAsWork` dumps the full `EpochAndRecordsMap.RecordsAndEpoch` (including the entire ConsumerRecord list) once per dropped batch. The resulting line is long enough to be truncated by common log aggregators (Datadog, Splunk, etc.), which hides the actually useful signal (which topic-partition was dropped, how many records, at what epoch). Split the log into: - WARN: concise `topic-partition (N records, epoch E)` -- always readable and grep-friendly, survives log-tooling truncation. - DEBUG (guarded by `log.isDebugEnabled()`): the full RecordsAndEpoch for deep troubleshooting when operators opt in. Behaviour, counts, and no-op semantics are unchanged; only the log output is restructured. This matches the approach already used elsewhere in the codebase where detailed state is logged at DEBUG. Fixes #631 --- .../state/RemovedPartitionState.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/RemovedPartitionState.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/RemovedPartitionState.java index 37d356d14..e084cbc1c 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/RemovedPartitionState.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/RemovedPartitionState.java @@ -64,8 +64,18 @@ public TopicPartition getTp() { @Override public void maybeRegisterNewPollBatchAsWork(@NonNull EpochAndRecordsMap.RecordsAndEpoch recordsAndEpoch) { - // no-op - log.warn("Dropping polled record batch for partition no longer assigned. WC: {}", recordsAndEpoch); + // no-op -- the partition is no longer assigned to this consumer, so the polled batch is dropped. + // Keep WARN concise (topic-partition + size + epoch) so it stays readable when many partitions are + // dropped at once (e.g. during rebalances) and isn't truncated by downstream log tooling. The full + // RecordsAndEpoch (which can be large) is emitted separately at DEBUG for detailed troubleshooting. + // See https://github.com/confluentinc/parallel-consumer/issues/631 + log.warn("Dropping polled record batch for partition no longer assigned: {} ({} records, epoch {})", + recordsAndEpoch.getTopicPartition(), + recordsAndEpoch.getRecords().size(), + recordsAndEpoch.getEpochOfPartitionAtPoll()); + if (log.isDebugEnabled()) { + log.debug("Full dropped batch for {}: {}", recordsAndEpoch.getTopicPartition(), recordsAndEpoch); + } } /**