diff --git a/Plugins/BenchmarkTool/BenchmarkTool+Baselines.swift b/Plugins/BenchmarkTool/BenchmarkTool+Baselines.swift index e553570d..8dd2f524 100644 --- a/Plugins/BenchmarkTool/BenchmarkTool+Baselines.swift +++ b/Plugins/BenchmarkTool/BenchmarkTool+Baselines.swift @@ -485,7 +485,7 @@ extension BenchmarkBaseline: Equatable { public func failsAbsoluteThresholdChecks( benchmarks: [Benchmark], p90Thresholds: [BenchmarkIdentifier: - [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]] + [BenchmarkMetric: BenchmarkThreshold]] ) -> BenchmarkResult.ThresholdDeviations { var allDeviationResults = BenchmarkResult.ThresholdDeviations() diff --git a/Plugins/BenchmarkTool/BenchmarkTool+Export+JMHFormatter.swift b/Plugins/BenchmarkTool/BenchmarkTool+Export+JMHFormatter.swift index 15d00576..8c5ef5d3 100644 --- a/Plugins/BenchmarkTool/BenchmarkTool+Export+JMHFormatter.swift +++ b/Plugins/BenchmarkTool/BenchmarkTool+Export+JMHFormatter.swift @@ -34,7 +34,7 @@ extension JMHPrimaryMetric { let factor = result.metric.countable == false ? 1_000 : 1 for p in percentiles { - percentileValues[String(p)] = Statistics.roundToDecimalplaces( + percentileValues[String(p)] = Statistics.roundToDecimalPlaces( Double(histogram.valueAtPercentile(p)) / Double(factor), 3 ) @@ -42,15 +42,15 @@ extension JMHPrimaryMetric { for value in histogram.recordedValues() { for _ in 0.. $1.uxPriority }) - let relativeResults = deviationResults.filter { - $0.name == nameAndTarget.name && $0.target == nameAndTarget.target && $0.metric == metric - && $0.relative == true - } - let absoluteResults = deviationResults.filter { - $0.name == nameAndTarget.name && $0.target == nameAndTarget.target && $0.metric == metric - && $0.relative == false - } let width = 40 - let percentileWidth = 15 + let percentileWidthFor4Columns = 15 + let percentileWidthFor3Columns = 20 + + let table = TextTable { + var columns: [Column] = [] + columns.reserveCapacity(4) + + let sign = + switch $0.deviation { + case .absolute: "Δ" + case .relative: "%" + case .range: "↔" + } + let unitDescription = metric.countable ? $0.units.description : $0.units.timeDescription + columns.append( + Column( + title: "\(metric.description) (\(unitDescription), \(sign))", + value: $0.percentile, + width: width, + align: .left + ) + ) - // The baseValue is the new baseline that we're using as the comparison base, so... - if absoluteResults.isEmpty == false { - let absoluteTable = TextTable { - [ - Column( - title: - "\(metric.description) (\(metric.countable ? $0.units.description : $0.units.timeDescription), Δ)", - value: $0.percentile, - width: width, - align: .left - ), + let baseValue = $0.baseValue + func baselineColumn(percentileWidth: Int) -> Column { + Column( + title: "\(comparingBaselineName)", + value: baseValue, + width: percentileWidth, + align: .right + ) + } + + // If absolute or relative add their columns together + var comparisonValue: Int? + var difference: String? + var tolerance: String? + switch $0.deviation { + case .absolute(let compareTo, let diff, let tol): + comparisonValue = compareTo + difference = diff.description + tolerance = tol.description + case .relative(let compareTo, let diff, let tol): + comparisonValue = compareTo + difference = Statistics.roundToDecimalPlaces(diff, 1).description + tolerance = Statistics.roundToDecimalPlaces(tol, 1).description + case .range: + break + } + + if let comparisonValue = comparisonValue, + let difference = difference, + let tolerance = tolerance + { + columns.append(contentsOf: [ Column( title: "\(baselineName)", - value: $0.comparisonValue, - width: percentileWidth, + value: comparisonValue, + width: percentileWidthFor4Columns, align: .right ), + baselineColumn(percentileWidth: percentileWidthFor4Columns), Column( - title: "\(comparingBaselineName)", - value: $0.baseValue, - width: percentileWidth, + title: "Difference \(sign)", + value: difference, + width: percentileWidthFor4Columns, align: .right ), - Column(title: "Difference Δ", value: $0.difference, width: percentileWidth, align: .right), Column( - title: "Threshold Δ", - value: $0.differenceThreshold, - width: percentileWidth, + title: "Tolerance \(sign)", + value: tolerance, + width: percentileWidthFor4Columns, align: .right ), - ] + ]) } - absoluteTable.print(absoluteResults, style: format.tableStyle) - } - - if relativeResults.isEmpty == false { - let relativeTable = TextTable { - [ - Column( - title: - "\(metric.description) (\(metric.countable ? $0.units.description : $0.units.timeDescription), %)", - value: $0.percentile, - width: width, - align: .left - ), + // Otherwise if range, then handle it alone + if case .range(let min, let max) = $0.deviation { + columns.append(contentsOf: [ + baselineColumn(percentileWidth: percentileWidthFor3Columns), Column( - title: "\(baselineName)", - value: $0.comparisonValue, - width: percentileWidth, + title: "Minimum", + value: min, + width: percentileWidthFor3Columns, align: .right ), Column( - title: "\(comparingBaselineName)", - value: $0.baseValue, - width: percentileWidth, + title: "Maximum", + value: max, + width: percentileWidthFor3Columns, align: .right ), - Column(title: "Difference %", value: $0.difference, width: percentileWidth, align: .right), - Column( - title: "Threshold %", - value: $0.differenceThreshold, - width: percentileWidth, - align: .right - ), - ] + ]) } - relativeTable.print(relativeResults, style: format.tableStyle) + return columns } + + table.print(filteredDeviations, style: format.tableStyle) } } } diff --git a/Plugins/BenchmarkTool/BenchmarkTool+ReadP90AbsoluteThresholds.swift b/Plugins/BenchmarkTool/BenchmarkTool+ReadP90AbsoluteThresholds.swift index d408a252..477d00e8 100644 --- a/Plugins/BenchmarkTool/BenchmarkTool+ReadP90AbsoluteThresholds.swift +++ b/Plugins/BenchmarkTool/BenchmarkTool+ReadP90AbsoluteThresholds.swift @@ -35,7 +35,7 @@ extension BenchmarkTool { static func makeBenchmarkThresholds( path: String, benchmarkIdentifier: BenchmarkIdentifier - ) -> [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]? { + ) -> [BenchmarkMetric: BenchmarkThreshold]? { var path = FilePath(path) if path.isAbsolute { path.append("\(benchmarkIdentifier.target).\(benchmarkIdentifier.name).p90.json") @@ -46,8 +46,7 @@ extension BenchmarkTool { path = cwdPath } - var p90Thresholds: [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold] = [:] - var p90ThresholdsRaw: [String: BenchmarkThresholds.AbsoluteThreshold]? + var p90Thresholds: [BenchmarkMetric: BenchmarkThreshold] = [:] do { let fileDescriptor = try FileDescriptor.open(path, .readOnly, options: [], permissions: .ownerRead) @@ -68,19 +67,11 @@ extension BenchmarkTool { readBytes.append(contentsOf: nextBytes) } - p90ThresholdsRaw = try JSONDecoder() + p90Thresholds = try JSONDecoder() .decode( - [String: BenchmarkThresholds.AbsoluteThreshold].self, + [BenchmarkMetric: BenchmarkThreshold].self, from: Data(readBytes) ) - - if let p90ThresholdsRaw { - p90ThresholdsRaw.forEach { metric, threshold in - if let metric = BenchmarkMetric(argument: metric) { - p90Thresholds[metric] = threshold - } - } - } } catch { print( "Failed to read file at \(path) [\(String(reflecting: error))] \(Errno(rawValue: errno).description)" diff --git a/Plugins/BenchmarkTool/BenchmarkTool+Thresholds.swift b/Plugins/BenchmarkTool/BenchmarkTool+Thresholds.swift index 3173b918..c343c39c 100644 --- a/Plugins/BenchmarkTool/BenchmarkTool+Thresholds.swift +++ b/Plugins/BenchmarkTool/BenchmarkTool+Thresholds.swift @@ -16,16 +16,17 @@ private let percentileWidth = 20 private let maxDescriptionWidth = 40 private struct ThresholdsTableEntry { + enum Values { + case absolute(p90: Int, absoluteTolerance: Int, relativeTolerance: Double) + case relativeOrRange(BenchmarkThreshold.RelativeOrRange) + } + var description: String - var p90: Int - var absolute: Int - var relative: Double + var value: Values } extension BenchmarkTool { - func printThresholds( - _ staticThresholdsPerBenchmark: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]] - ) { + func printThresholds(_ staticThresholdsPerBenchmark: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThreshold]]) { guard !staticThresholdsPerBenchmark.isEmpty else { print("No thresholds defined.") @@ -35,13 +36,44 @@ extension BenchmarkTool { print("") var tableEntries: [ThresholdsTableEntry] = [] - let table = TextTable { - [ - Column(title: "Metric", value: "\($0.description)", width: maxDescriptionWidth, align: .left), - Column(title: "Threshold .p90", value: $0.p90, width: percentileWidth, align: .right), - Column(title: "Allowed %", value: $0.relative, width: percentileWidth, align: .right), - Column(title: "Allowed Δ", value: $0.absolute, width: percentileWidth, align: .right), - ] + let table = TextTable { entry in + var columns: [Column] = [] + columns.reserveCapacity(4) + + columns.append( + Column(title: "Metric", value: entry.description, width: maxDescriptionWidth, align: .left), + ) + + switch entry.value { + case .absolute(let p90, let absoluteTolerance, let relativeTolerance): + columns.append(contentsOf: [ + Column(title: "Threshold .p90", value: p90, width: percentileWidth, align: .right), + Column(title: "Allowed %", value: relativeTolerance, width: percentileWidth, align: .right), + Column(title: "Allowed Δ", value: absoluteTolerance, width: percentileWidth, align: .right), + ]) + case .relativeOrRange(let relativeOrRange): + relativeOrRange.preconditionContainsAnyValue() + + if let relative = relativeOrRange.relative { + let tolerancePercentage = Statistics.roundToDecimalPlaces(relative.tolerancePercentage, 1) + columns.append(contentsOf: [ + Column( + title: "Allowed %", + value: "\(relative.base) ± \(tolerancePercentage)%", + width: percentileWidth, + align: .right + ) + ]) + } + if let range = relativeOrRange.range { + columns.append(contentsOf: [ + Column(title: "Allowed min", value: "\(range.min)", width: percentileWidth, align: .right), + Column(title: "Allowed max", value: "\(range.max)", width: percentileWidth, align: .right), + ]) + } + } + + return columns } staticThresholdsPerBenchmark.forEach { benchmarkIdentifier, staticThresholds in @@ -50,25 +82,34 @@ extension BenchmarkTool { let thresholdDeviations = benchmarks.first(where: { benchmarkIdentifier - == .init( - target: $0.target, - name: $0.name - ) + == .init(target: $0.target, name: $0.name) })? .configuration.thresholds ?? .init() staticThresholds.forEach { threshold in - let absoluteThreshold = thresholdDeviations[threshold.key]?.absolute[.p90] ?? 0 - let relativeThreshold = thresholdDeviations[threshold.key]?.relative[.p90] ?? 0 - - tableEntries.append( - .init( - description: threshold.key.description, - p90: threshold.value, - absolute: absoluteThreshold, - relative: relativeThreshold + switch threshold.value { + case .absolute(let value): + let absoluteThreshold = thresholdDeviations[threshold.key]?.absolute[.p90] ?? 0 + let relativeThreshold = thresholdDeviations[threshold.key]?.relative[.p90] ?? 0 + + tableEntries.append( + .init( + description: threshold.key.description, + value: .absolute( + p90: value, + absoluteTolerance: absoluteThreshold, + relativeTolerance: relativeThreshold + ) + ) + ) + case .relativeOrRange(let relativeOrRange): + tableEntries.append( + .init( + description: threshold.key.description, + value: .relativeOrRange(relativeOrRange) + ) ) - ) + } } table.print(tableEntries, style: format.tableStyle) tableEntries = [] diff --git a/Sources/Benchmark/BenchmarkMetric.swift b/Sources/Benchmark/BenchmarkMetric.swift index b5d06096..b1032cc3 100644 --- a/Sources/Benchmark/BenchmarkMetric.swift +++ b/Sources/Benchmark/BenchmarkMetric.swift @@ -132,7 +132,7 @@ public extension BenchmarkMetric { return true case .objectAllocCount, .retainCount, .releaseCount, .retainReleaseDelta: return true - case let .custom(_, _, useScaleFactor): + case .custom(_, _, let useScaleFactor): return useScaleFactor default: return false @@ -144,7 +144,7 @@ public extension BenchmarkMetric { switch self { case .throughput: return .prefersLarger - case let .custom(_, polarity, _): + case .custom(_, let polarity, _): return polarity default: return .prefersSmaller @@ -213,7 +213,7 @@ public extension BenchmarkMetric { return "Δ" case .deltaPercentage: return "Δ %" - case let .custom(name, _, _): + case .custom(let name, _, _): return name } } @@ -417,7 +417,7 @@ public extension BenchmarkMetric { return "Δ" case .deltaPercentage: return "Δ %" - case let .custom(name, _, _): + case .custom(let name, _, _): return name } } @@ -491,4 +491,16 @@ public extension BenchmarkMetric { } } +/// `CodingKeyRepresentable` conformance enables Codable to encode/decode a dictionary with keys of +/// type `BenchmarkMetric`, as if the key type was `String` and not `BenchmarkMetric`. +extension BenchmarkMetric: CodingKeyRepresentable { + public var codingKey: any CodingKey { + self.rawDescription.codingKey + } + + public init?(codingKey: T) where T: CodingKey { + self.init(argument: codingKey.stringValue) + } +} + // swiftlint:enable cyclomatic_complexity function_body_length diff --git a/Sources/Benchmark/BenchmarkResult.swift b/Sources/Benchmark/BenchmarkResult.swift index 80350f2c..9e16c215 100644 --- a/Sources/Benchmark/BenchmarkResult.swift +++ b/Sources/Benchmark/BenchmarkResult.swift @@ -459,16 +459,43 @@ public struct BenchmarkResult: Codable, Comparable, Equatable { } public struct ThresholdDeviation { + public enum Deviation { + case absolute(comparedTo: Int, difference: Int, tolerance: Int) + case relative(comparedTo: Int, differencePercentage: Double, tolerancePercentage: Double) + case range(min: Int, max: Int) + + public var isRelative: Bool { + switch self { + case .relative: + return true + default: + return false + } + } + + public var uxPriority: Int { + switch self { + case .range: + return 2 + case .absolute: + return 1 + case .relative: + return 0 + } + } + } + public let name: String public let target: String public let metric: BenchmarkMetric public let percentile: BenchmarkResult.Percentile public let baseValue: Int - public let comparisonValue: Int - public let difference: Int - public let differenceThreshold: Int - public let relative: Bool + public let deviation: Deviation public let units: Statistics.Units + + public var uxPriority: Int { + return deviation.uxPriority + } } public struct ThresholdDeviations { @@ -490,56 +517,127 @@ public struct BenchmarkResult: Codable, Comparable, Equatable { func appendDeviationResultsFor( _ metric: BenchmarkMetric, _ lhs: Int, - _ rhs: Int, + /// Either absolute values or thresholds from static threshold files. + _ rhs: BenchmarkThreshold, _ percentile: Self.Percentile, - _ thresholds: BenchmarkThresholds, + /// Thresholds from 'configuration.thresholds' in the benchmark code. + _ thresholdsFromCode: BenchmarkThresholds, _ thresholdResults: inout ThresholdDeviations, _ name: String = "unknown name", _ target: String = "unknown target" ) { let reverseComparison = metric.polarity == .prefersLarger - let absoluteDifference = (reverseComparison ? -1 : 1) * (lhs - rhs) - let relativeDifference = - (reverseComparison ? 1 : -1) * (rhs != 0 ? (100 - (100.0 * Double(lhs) / Double(rhs))) : 0.0) - - if let threshold = thresholds.relative[percentile], !(-threshold...threshold).contains(relativeDifference) { - let deviation = ThresholdDeviation( - name: name, - target: target, - metric: metric, - percentile: percentile, - baseValue: normalize(lhs), - comparisonValue: normalize(rhs), - difference: Int(Statistics.roundToDecimalplaces(relativeDifference, 1)), - differenceThreshold: Int(threshold), - relative: true, - units: Statistics.Units(timeUnits) - ) - if relativeDifference > threshold { - thresholdResults.regressions.append(deviation) - } else if relativeDifference < -threshold { - thresholdResults.improvements.append(deviation) + switch rhs { + case .absolute(let rhs): + let absoluteDifference = (reverseComparison ? -1 : 1) * (lhs - rhs) + let relativeDifference = + (reverseComparison ? 1 : -1) * (rhs != 0 ? (100 - (100.0 * Double(lhs) / Double(rhs))) : 0.0) + + if let threshold = thresholdsFromCode.relative[percentile], + !(-threshold...threshold).contains(relativeDifference) + { + let deviation = ThresholdDeviation( + name: name, + target: target, + metric: metric, + percentile: percentile, + baseValue: normalize(lhs), + deviation: .relative( + comparedTo: normalize(rhs), + differencePercentage: Statistics.roundToDecimalPlaces(relativeDifference, 1), + tolerancePercentage: Statistics.roundToDecimalPlaces(threshold, 1) + ), + units: Statistics.Units(timeUnits) + ) + if relativeDifference > threshold { + thresholdResults.regressions.append(deviation) + } else if relativeDifference < -threshold { + thresholdResults.improvements.append(deviation) + } } - } - if let threshold = thresholds.absolute[percentile], !(-threshold...threshold).contains(absoluteDifference) { - let deviation = ThresholdDeviation( - name: name, - target: target, - metric: metric, - percentile: percentile, - baseValue: normalize(lhs), - comparisonValue: normalize(rhs), - difference: normalize(absoluteDifference), - differenceThreshold: normalize(threshold), - relative: false, - units: Statistics.Units(timeUnits) - ) + if let threshold = thresholdsFromCode.absolute[percentile], + !(-threshold...threshold).contains(absoluteDifference) + { + let deviation = ThresholdDeviation( + name: name, + target: target, + metric: metric, + percentile: percentile, + baseValue: normalize(lhs), + deviation: .absolute( + comparedTo: normalize(rhs), + difference: normalize(absoluteDifference), + tolerance: normalize(threshold) + ), + units: Statistics.Units(timeUnits) + ) + if absoluteDifference > threshold { + thresholdResults.regressions.append(deviation) + } else if absoluteDifference < -threshold { + thresholdResults.improvements.append(deviation) + } + } + + case .relativeOrRange(let rhs): + if thresholdsFromCode.definitelyContainsUserSpecifiedThresholds(at: percentile) { + print( + """ + Warning: Static threshold files contain relative or range thresholds for metric '\(metric)' at + percentile '\(percentile)', but 'configuration.thresholds' also contains threshold tolerance for this metric. + Will ignore 'configuration.thresholds'. + To silence this warning, remove 'configuration.thresholds' from your benchmark code of this benchmark. + + """ + ) + } + + if let relative = rhs.relative { + let relativeComparison = relative.contains(lhs) + if !relativeComparison.contains { + let relativeDifference = (reverseComparison ? 1 : -1) * relativeComparison.deviation + let deviation = ThresholdDeviation( + name: name, + target: target, + metric: metric, + percentile: percentile, + baseValue: normalize(lhs), + deviation: .relative( + comparedTo: normalize(relative.base), + differencePercentage: Statistics.roundToDecimalPlaces(relativeDifference, 1), + tolerancePercentage: Statistics.roundToDecimalPlaces( + relative.tolerancePercentage, + 1 + ) + ), + units: Statistics.Units(timeUnits) + ) + if relativeDifference > relative.tolerancePercentage { + thresholdResults.regressions.append(deviation) + } else if relativeDifference < -relative.tolerancePercentage { + thresholdResults.improvements.append(deviation) + } + } + } - if absoluteDifference > threshold { - thresholdResults.regressions.append(deviation) - } else if absoluteDifference < -threshold { - thresholdResults.improvements.append(deviation) + if let range = rhs.range, !range.contains(lhs) { + let deviation = ThresholdDeviation( + name: name, + target: target, + metric: metric, + percentile: percentile, + baseValue: normalize(lhs), + deviation: .range( + min: normalize(range.min), + max: normalize(range.max) + ), + units: Statistics.Units(timeUnits) + ) + if lhs < range.min { + thresholdResults.regressions.append(deviation) + } else if lhs > range.max { + thresholdResults.improvements.append(deviation) + } } } } @@ -564,7 +662,7 @@ public struct BenchmarkResult: Codable, Comparable, Equatable { appendDeviationResultsFor( lhs.metric, lhsPercentiles[percentile], - rhsPercentiles[percentile], + .absolute(rhsPercentiles[percentile]), Self.Percentile(rawValue: percentile)!, thresholds, &thresholdResults, @@ -579,7 +677,7 @@ public struct BenchmarkResult: Codable, Comparable, Equatable { // Absolute checks for --check-absolute, just check p90 public func deviationsAgainstAbsoluteThresholds( thresholds: BenchmarkThresholds, - p90Threshold: BenchmarkThresholds.AbsoluteThreshold, + p90Threshold: BenchmarkThreshold, name: String = "test", target: String = "test" ) -> ThresholdDeviations { diff --git a/Sources/Benchmark/BenchmarkRunner.swift b/Sources/Benchmark/BenchmarkRunner.swift index 542edb78..b3f2ff3e 100644 --- a/Sources/Benchmark/BenchmarkRunner.swift +++ b/Sources/Benchmark/BenchmarkRunner.swift @@ -149,7 +149,7 @@ public struct BenchmarkRunner: AsyncParsableCommand, BenchmarkRunnerReadWrite { } try channel.write(.end) - case let .run(benchmarkToRun): + case .run(let benchmarkToRun): benchmark = Benchmark.benchmarks.first { $0.name == benchmarkToRun.name } if let benchmark { diff --git a/Sources/Benchmark/BenchmarkThresholds.swift b/Sources/Benchmark/BenchmarkThresholds.swift index 699df0ba..69db5d6c 100644 --- a/Sources/Benchmark/BenchmarkThresholds.swift +++ b/Sources/Benchmark/BenchmarkThresholds.swift @@ -32,3 +32,175 @@ public struct BenchmarkThresholds: Codable { public let relative: RelativeThresholds public let absolute: AbsoluteThresholds } + +extension BenchmarkThresholds { + public func definitelyContainsUserSpecifiedThresholds(at percentile: BenchmarkResult.Percentile) -> Bool { + let defaultCodeThresholds = BenchmarkThresholds.default + let relative = self.relative[percentile] + let absolute = self.absolute[percentile] + var relativeNonDefaultThresholdsExist: Bool { + (relative ?? 0) != 0 + && relative != defaultCodeThresholds.relative[percentile] + } + var absoluteNonDefaultThresholdsExist: Bool { + (absolute ?? 0) != 0 + && absolute != defaultCodeThresholds.absolute[percentile] + } + return relativeNonDefaultThresholdsExist || absoluteNonDefaultThresholdsExist + } +} + +public enum BenchmarkThreshold: Codable { + public struct RelativeOrRange: Codable { + public struct Relative: Encodable { + public let base: Int + public let tolerancePercentage: Double + + init(base: Int, tolerancePercentage: Double) { + precondition(base > 0, "base must be positive") + precondition(tolerancePercentage > 0, "tolerancePercentage must be positive") + self.base = base + self.tolerancePercentage = tolerancePercentage + } + + /// Returns whether or not the value satisfies this relative range, as well as the + /// percentage of the deviation of the value. + public func contains(_ value: Int) -> (contains: Bool, deviation: Double) { + let deviation = Double(value - base) / Double(base) * 100 + return (abs(deviation) <= tolerancePercentage, deviation) + } + } + + public struct Range: Encodable { + public let min: Int + public let max: Int + + init(min: Int, max: Int) { + precondition(min <= max, "min must be less than or equal to max") + self.min = min + self.max = max + } + + public func contains(_ value: Int) -> Bool { + return value >= min && value <= max + } + } + + public let relative: Relative? + public let range: Range? + + init(relative: Relative?, range: Range?) { + self.relative = relative + self.range = range + preconditionContainsAnyValue() + } + + public func preconditionContainsAnyValue() { + precondition( + self.containsAnyValue, + "RelativeOrRange must contain either a relative or range, but contains neither" + ) + } + + var containsAnyValue: Bool { + self.relative != nil || self.range != nil + } + + enum CodingKeys: String, CodingKey { + case base + case tolerancePercentage + case min + case max + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + + let base = try container.decodeIfPresent(Int.self, forKey: .base) + let tolerancePercentage = try container.decodeIfPresent(Double.self, forKey: .tolerancePercentage) + let min = try container.decodeIfPresent(Int.self, forKey: .min) + let max = try container.decodeIfPresent(Int.self, forKey: .max) + + var relative: Relative? + var range: Range? + + if let base, let tolerancePercentage { + relative = Relative(base: base, tolerancePercentage: tolerancePercentage) + + guard base > 0, tolerancePercentage > 0 else { + throw DecodingError.dataCorrupted( + .init( + codingPath: decoder.codingPath, + debugDescription: """ + RelativeOrRange thresholds object contains an invalid relative values. + 'base' (\(base)) and 'tolerancePercentage' (\(tolerancePercentage)) must be positive. + """ + ) + ) + } + } + if let min, let max { + range = Range(min: min, max: max) + + guard min <= max else { + throw DecodingError.dataCorrupted( + .init( + codingPath: decoder.codingPath, + debugDescription: """ + RelativeOrRange thresholds object contains invalid min-max values. + 'min' (\(min)) and max ('\(max)') don't satisfy the requirements of min <= max. + """ + ) + ) + } + } + + self.relative = relative + self.range = range + + if !self.containsAnyValue { + throw DecodingError.dataCorrupted( + .init( + codingPath: decoder.codingPath, + debugDescription: """ + RelativeOrRange thresholds object does not contain either a valid relative or range. + For relative thresholds, both 'base' (Int) and 'tolerancePercentage' (Double) must be present and valid. + For range thresholds, both 'min' (Int) and 'max' (Int) must be present and valid. + You can declare both relative and range in the same object together, or just one of them. + Example: { "min": 90, "max": 110 } + Example: { "base": 115, "tolerancePercentage": 5.5 } + Example: { "base": 115, "tolerancePercentage": 4.5, "min": 90, "max": 110 } + """ + ) + ) + } + } + + public func encode(to encoder: any Encoder) throws { + try self.relative?.encode(to: encoder) + try self.range?.encode(to: encoder) + } + } + + case absolute(Int) + case relativeOrRange(RelativeOrRange) + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if let value = try? container.decode(Int.self) { + self = .absolute(value) + } else { + let value = try RelativeOrRange(from: decoder) + self = .relativeOrRange(value) + } + } + + public func encode(to encoder: Encoder) throws { + switch self { + case .absolute(let value): + try value.encode(to: encoder) + case .relativeOrRange(let value): + try value.encode(to: encoder) + } + } +} diff --git a/Sources/Benchmark/Statistics.swift b/Sources/Benchmark/Statistics.swift index 49dec756..d0b0264c 100644 --- a/Sources/Benchmark/Statistics.swift +++ b/Sources/Benchmark/Statistics.swift @@ -181,7 +181,7 @@ public final class Statistics: Codable { } // Rounds decimals for display - public static func roundToDecimalplaces(_ original: Double, _ decimals: Int = 2) -> Double { + public static func roundToDecimalPlaces(_ original: Double, _ decimals: Int = 2) -> Double { let factor: Double = .pow(10.0, Double(decimals)) var original: Double = original * factor original.round(.toNearestOrEven) diff --git a/Tests/BenchmarkTests/BenchmarkResultTests.swift b/Tests/BenchmarkTests/BenchmarkResultTests.swift index 46170551..ee6cb29f 100644 --- a/Tests/BenchmarkTests/BenchmarkResultTests.swift +++ b/Tests/BenchmarkTests/BenchmarkResultTests.swift @@ -379,21 +379,21 @@ final class BenchmarkResultTests: XCTestCase { Benchmark.checkAbsoluteThresholds = true deviations = thirdResult.deviationsAgainstAbsoluteThresholds( thresholds: absoluteThresholdsP90, - p90Threshold: 1_497 + p90Threshold: .absolute(1_497) ) XCTAssertFalse(deviations.regressions.isEmpty) Benchmark.checkAbsoluteThresholds = true deviations = thirdResult.deviationsAgainstAbsoluteThresholds( thresholds: absoluteThresholdsP90, - p90Threshold: 1_505 + p90Threshold: .absolute(1_505) ) XCTAssertFalse(deviations.improvements.isEmpty) Benchmark.checkAbsoluteThresholds = true deviations = thirdResult.deviationsAgainstAbsoluteThresholds( thresholds: absoluteThresholdsP90, - p90Threshold: 1_501 + p90Threshold: .absolute(1_501) ) XCTAssertTrue(deviations.improvements.isEmpty && deviations.regressions.isEmpty) } diff --git a/Tests/BenchmarkTests/StatisticsTests.swift b/Tests/BenchmarkTests/StatisticsTests.swift index f330a15c..50d26c0b 100644 --- a/Tests/BenchmarkTests/StatisticsTests.swift +++ b/Tests/BenchmarkTests/StatisticsTests.swift @@ -26,7 +26,7 @@ final class StatisticsTests: XCTestCase { stats.add(measurement) } - XCTAssertEqual(Statistics.roundToDecimalplaces(123.4567898972239487234), 123.46) + XCTAssertEqual(Statistics.roundToDecimalPlaces(123.4567898972239487234), 123.46) XCTAssertEqual(stats.measurementCount, measurementCount * 2) XCTAssertEqual(stats.units(), .count) XCTAssertEqual(round(stats.histogram.mean), round(Double(measurementCount / 2)))