diff --git a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala index 648532faa3a1c..1e80eb66dc520 100644 --- a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala +++ b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala @@ -110,10 +110,12 @@ case object GarbageCollectionMetrics extends ExecutorMetricType with Logging { "MinorGCTime", "MajorGCCount", "MajorGCTime", - "TotalGCTime" + "TotalGCTime", + "ConcurrentGCCount", + "ConcurrentGCTime" ) - /* We builtin some common GC collectors which categorized as young generation and old */ + /* We builtin some common GC collectors */ private[spark] val YOUNG_GENERATION_BUILTIN_GARBAGE_COLLECTORS = Seq( "Copy", "PS Scavenge", @@ -128,6 +130,8 @@ case object GarbageCollectionMetrics extends ExecutorMetricType with Logging { "G1 Old Generation" ) + private[spark] val BUILTIN_CONCURRENT_GARBAGE_COLLECTOR = "G1 Concurrent GC" + private lazy val youngGenerationGarbageCollector: Seq[String] = { SparkEnv.get.conf.get(config.EVENT_LOG_GC_METRICS_YOUNG_GENERATION_GARBAGE_COLLECTORS) } @@ -147,6 +151,9 @@ case object GarbageCollectionMetrics extends ExecutorMetricType with Logging { } else if (oldGenerationGarbageCollector.contains(mxBean.getName)) { gcMetrics(2) = mxBean.getCollectionCount gcMetrics(3) = mxBean.getCollectionTime + } else if (BUILTIN_CONCURRENT_GARBAGE_COLLECTOR.equals(mxBean.getName)) { + gcMetrics(5) = mxBean.getCollectionCount + gcMetrics(6) = mxBean.getCollectionTime } else if (!nonBuiltInCollectors.contains(mxBean.getName)) { nonBuiltInCollectors = mxBean.getName +: nonBuiltInCollectors // log it when first seen diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala index 9658e5e627724..ca088dc80550b 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala @@ -97,10 +97,10 @@ private[v1] class PrometheusResource extends ApiRequestContext { names.foreach { name => sb.append(s"$prefix${name}_bytes$labels ${m.getMetricValue(name)}\n") } - Seq("MinorGCCount", "MajorGCCount").foreach { name => + Seq("MinorGCCount", "MajorGCCount", "ConcurrentGCCount").foreach { name => sb.append(s"$prefix${name}_total$labels ${m.getMetricValue(name)}\n") } - Seq("MinorGCTime", "MajorGCTime").foreach { name => + Seq("MinorGCTime", "MajorGCTime", "ConcurrentGCTime").foreach { name => sb.append(s"$prefix${name}_seconds_total$labels ${m.getMetricValue(name) * 0.001}\n") } } diff --git a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json index 850c3777ec4d8..ac0f2ce26051d 100644 --- a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json @@ -76,7 +76,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } }, { "status" : "COMPLETE", @@ -156,7 +158,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } }, { "status" : "COMPLETE", @@ -236,6 +240,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json index ed4ed9ad87185..d614bb000e4ce 100644 --- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json @@ -887,7 +887,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : true }, @@ -928,7 +930,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -956,6 +960,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json index f96a59fae5378..475dee00a2654 100644 --- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json @@ -1021,7 +1021,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : true }, @@ -1062,7 +1064,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : true }, @@ -1103,7 +1107,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -1144,7 +1150,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -1185,7 +1193,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : true } @@ -1213,6 +1223,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json index ec3fc280b0a5e..a860682ca2e24 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json @@ -42,7 +42,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json index 9b7498d9e9145..2833cdcfde5dd 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json @@ -48,7 +48,9 @@ "MinorGCTime" : 55, "MajorGCCount" : 3, "MajorGCTime" : 144, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -108,7 +110,9 @@ "MinorGCTime" : 145, "MajorGCCount" : 2, "MajorGCTime" : 63, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { "NM_HTTP_ADDRESS" : "test-3.vpc.company.com:8042", @@ -178,7 +182,9 @@ "MinorGCTime" : 106, "MajorGCCount" : 2, "MajorGCTime" : 75, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { "NM_HTTP_ADDRESS" : "test-4.vpc.company.com:8042", @@ -248,7 +254,9 @@ "MinorGCTime" : 140, "MajorGCCount" : 2, "MajorGCTime" : 60, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { "NM_HTTP_ADDRESS" : "test-2.vpc.company.com:8042", diff --git a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json index fbb7b6631f02a..8a96858a2014a 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json @@ -85,7 +85,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -145,7 +147,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -205,7 +209,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -265,7 +271,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json index fbb7b6631f02a..8a96858a2014a 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json @@ -85,7 +85,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -145,7 +147,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -205,7 +209,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -265,7 +271,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json index b72ed0a625420..0e5e73f36fabd 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json @@ -73,7 +73,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -127,7 +129,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -181,7 +185,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -235,7 +241,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json index fee7377f18134..dc1bcd6a39625 100644 --- a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json @@ -75,7 +75,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json index 9e390a995c36c..e24ac4f82b8a8 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json @@ -92,7 +92,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -120,6 +122,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json index 887d2678e6160..659e3c41d9289 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json @@ -595,7 +595,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -625,6 +627,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json index 3bb59aaf5b507..f84cf26fcf1d6 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json @@ -595,7 +595,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -625,6 +627,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json index b688b72b04d50..564f3eadd1cc2 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json @@ -597,7 +597,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -625,6 +627,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json index 83ffb7da8e77f..2bf7f34803775 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json @@ -721,7 +721,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -749,7 +751,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json index e3cd980943450..8df41bfcc8d78 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json @@ -74,7 +74,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -155,7 +157,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -235,7 +239,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -315,7 +321,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json index e4caffcf10787..730df3fbd5341 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json @@ -78,7 +78,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json index d3459be777d48..16d92244ee0d0 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json @@ -74,7 +74,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -155,7 +157,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -236,7 +240,9 @@ "MinorGCTime" : 33, "MajorGCCount" : 3, "MajorGCTime" : 110, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json index 3880818a7b5df..e38741b7bf6e7 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json @@ -639,7 +639,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -667,7 +669,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json index d3eb7d55e0e1d..630b0512e8f98 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json @@ -1147,7 +1147,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -1188,7 +1190,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -1216,7 +1220,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json index 3ad18f816fbe5..23770480ad62a 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json @@ -424,7 +424,9 @@ "MinorGCTime" : 280, "MajorGCCount" : 2, "MajorGCTime" : 1116, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -465,7 +467,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -506,7 +510,9 @@ "MinorGCTime" : 587, "MajorGCCount" : 2, "MajorGCTime" : 906, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -547,7 +553,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -584,6 +592,8 @@ "MinorGCTime" : 587, "MajorGCCount" : 2, "MajorGCTime" : 1116, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json index c89b82caf3818..c8458a409589e 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json @@ -1147,7 +1147,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -1188,7 +1190,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -1216,7 +1220,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "taskMetricsDistributions" : { "quantiles" : [ 0.0, 0.25, 0.5, 0.75, 1.0 ], @@ -1306,7 +1312,9 @@ "MinorGCTime" : [ 0.0, 0.0, 115.0, 115.0, 115.0 ], "MajorGCCount" : [ 0.0, 0.0, 4.0, 4.0, 4.0 ], "MajorGCTime" : [ 0.0, 0.0, 339.0, 339.0, 339.0 ], - "TotalGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ] + "TotalGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], + "ConcurrentGCCount" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], + "ConcurrentGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ] } }, "isShufflePushEnabled" : false, diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala index 8af8c8579232f..cdf50c8b16539 100644 --- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala @@ -58,21 +58,21 @@ class JsonProtocolSuite extends SparkFunSuite { makeTaskInfo(123L, 234, 67, 234, 345L, false), new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, - 0, 0, 0, 0, 80001L)), + 0, 0, 0, 0, 80001L, 3, 3)), makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, 0, hasHadoopInput = false, hasOutput = false)) val taskEndWithHadoopInput = SparkListenerTaskEnd(1, 0, "ShuffleMapTask", Success, makeTaskInfo(123L, 234, 67, 234, 345L, false), new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, - 0, 0, 0, 0, 80001L)), + 0, 0, 0, 0, 80001L, 3, 3)), makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, 0, hasHadoopInput = true, hasOutput = false)) val taskEndWithOutput = SparkListenerTaskEnd(1, 0, "ResultTask", Success, makeTaskInfo(123L, 234, 67, 234, 345L, false), new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, - 0, 0, 0, 0, 80001L)), + 0, 0, 0, 0, 80001L, 3, 3)), makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, 0, hasHadoopInput = true, hasOutput = true)) val jobStart = { @@ -136,7 +136,7 @@ class JsonProtocolSuite extends SparkFunSuite { val executorUpdates = new ExecutorMetrics( Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, - 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L)) + 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L, 3, 3)) SparkListenerExecutorMetricsUpdate("exec3", Seq((1L, 2, 3, accumUpdates)), Map((0, 0) -> executorUpdates)) } @@ -147,7 +147,7 @@ class JsonProtocolSuite extends SparkFunSuite { SparkListenerStageExecutorMetrics("1", 2, 3, new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, - 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L))) + 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L, 3, 3))) val rprofBuilder = new ResourceProfileBuilder() val taskReq = new TaskResourceRequests() .cpus(1) @@ -1784,7 +1784,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime" : 0, | "MajorGCCount" : 0, | "MajorGCTime" : 0, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | }, | "Task Metrics": { | "Executor Deserialize Time": 300, @@ -1923,7 +1925,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime" : 0, | "MajorGCCount" : 0, | "MajorGCTime" : 0, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | }, | "Task Metrics": { | "Executor Deserialize Time": 300, @@ -2062,7 +2066,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime" : 0, | "MajorGCCount" : 0, | "MajorGCTime" : 0, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | }, | "Task Metrics": { | "Executor Deserialize Time": 300, @@ -2963,7 +2969,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime": 90, | "MajorGCCount": 2, | "MajorGCTime": 20, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | } | } | ] @@ -2998,7 +3006,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime": 90, | "MajorGCCount": 2, | "MajorGCTime": 20, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | } |} """.stripMargin