From da938d1e1fcaee7c19438f62d8019fffb18e445c Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Thu, 18 Dec 2025 10:01:32 +0100 Subject: [PATCH 1/3] Make coverage support incremental compilation --- .../tools/dotc/coverage/Serializer.scala | 67 ++++++++++++++++++- .../dotc/transform/InstrumentCoverage.scala | 27 +++++++- 2 files changed, 89 insertions(+), 5 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/coverage/Serializer.scala b/compiler/src/dotty/tools/dotc/coverage/Serializer.scala index de9c29965ded..d46ef46479df 100644 --- a/compiler/src/dotty/tools/dotc/coverage/Serializer.scala +++ b/compiler/src/dotty/tools/dotc/coverage/Serializer.scala @@ -1,22 +1,27 @@ package dotty.tools.dotc package coverage +import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.{Path, Paths, Files} import java.io.Writer import scala.collection.mutable.StringBuilder +import scala.io.Source /** * Serializes scoverage data. - * @see https://github.com/scoverage/scalac-scoverage-plugin/blob/main/scalac-scoverage-plugin/src/main/scala/scoverage/Serializer.scala + * @see https://github.com/scoverage/scalac-scoverage-plugin/blob/main/serializer/src/main/scala/scoverage/serialize/Serializer.scala */ object Serializer: private val CoverageFileName = "scoverage.coverage" private val CoverageDataFormatVersion = "3.0" + def coverageFilePath(dataDir: String) = + Paths.get(dataDir, CoverageFileName).toAbsolutePath + /** Write out coverage data to the given data directory, using the default coverage filename */ def serialize(coverage: Coverage, dataDir: String, sourceRoot: String): Unit = - serialize(coverage, Paths.get(dataDir, CoverageFileName).toAbsolutePath, Paths.get(sourceRoot).toAbsolutePath) + serialize(coverage, coverageFilePath(dataDir), Paths.get(sourceRoot).toAbsolutePath) /** Write out coverage data to a file. */ def serialize(coverage: Coverage, file: Path, sourceRoot: Path): Unit = @@ -85,6 +90,64 @@ object Serializer: .sortBy(_.id) .foreach(stmt => writeStatement(stmt, writer)) + def deserialize(file: Path): Coverage = + val source = Source.fromFile(file.toFile(), UTF_8.name()) + try deserialize(source.getLines()) + finally source.close() + + def deserialize(lines: Iterator[String]): Coverage = + def toStatement(lines: Iterator[String]): Statement = + val id: Int = lines.next().toInt + val sourcePath = lines.next() + val packageName = lines.next() + val className = lines.next() + val classType = lines.next() + val fullClassName = lines.next() + val method = lines.next() + val loc = Location( + packageName, + className, + fullClassName, + classType, + method, + Paths.get(sourcePath) + ) + val start: Int = lines.next().toInt + val end: Int = lines.next().toInt + val lineNo: Int = lines.next().toInt + val symbolName: String = lines.next() + val treeName: String = lines.next() + val branch: Boolean = lines.next().toBoolean + val count: Int = lines.next().toInt + val ignored: Boolean = lines.next().toBoolean + val desc = lines.toList.mkString("\n") + Statement( + loc, + id, + start, + end, + lineNo, + desc, + symbolName, + treeName, + branch, + ignored + ) + + val headerFirstLine = lines.next() + require( + headerFirstLine == s"# Coverage data, format version: $CoverageDataFormatVersion", + "Wrong file format" + ) + + val linesWithoutHeader = lines.dropWhile(_.startsWith("#")) + val coverage = Coverage() + while !linesWithoutHeader.isEmpty do + val oneStatementLines = linesWithoutHeader.takeWhile(_ != "\f") + coverage.addStatement(toStatement(oneStatementLines)) + end while + coverage + /** Makes a String suitable for output in the coverage statement data as a single line. * Escaped characters: '\\' (backslash), '\n', '\r', '\f' */ diff --git a/compiler/src/dotty/tools/dotc/transform/InstrumentCoverage.scala b/compiler/src/dotty/tools/dotc/transform/InstrumentCoverage.scala index 689d7e01e0ae..a10c2a276a74 100644 --- a/compiler/src/dotty/tools/dotc/transform/InstrumentCoverage.scala +++ b/compiler/src/dotty/tools/dotc/transform/InstrumentCoverage.scala @@ -2,6 +2,7 @@ package dotty.tools.dotc package transform import java.io.File +import java.nio.file.Files import ast.tpd.* import collection.mutable @@ -41,6 +42,7 @@ class InstrumentCoverage extends MacroTransform with IdentityDenotTransformer: private var coverageExcludeClasslikePatterns: List[Pattern] = Nil private var coverageExcludeFilePatterns: List[Pattern] = Nil + private var lastCompiledFiles: Set[String] = Set.empty override def run(using ctx: Context): Unit = val outputPath = ctx.settings.coverageOutputDir.value @@ -50,12 +52,18 @@ class InstrumentCoverage extends MacroTransform with IdentityDenotTransformer: val newlyCreated = dataDir.mkdirs() if !newlyCreated then - // If the directory existed before, let's clean it up. + // If the directory existed before, clean measurement files. dataDir.listFiles - .filter(_.getName.startsWith("scoverage")) + .filter(_.getName.startsWith("scoverage.measurements.")) .foreach(_.delete()) end if + val coverageFilePath = Serializer.coverageFilePath(outputPath) + val previousCoverage = + if Files.exists(coverageFilePath) then + Serializer.deserialize(coverageFilePath) + else Coverage() + // Initialise a coverage object if it does not exist yet if ctx.base.coverage == null then ctx.base.coverage = Coverage() @@ -66,7 +74,19 @@ class InstrumentCoverage extends MacroTransform with IdentityDenotTransformer: ctx.base.coverage.nn.removeStatementsFromFile(ctx.compilationUnit.source.file.absolute.jpath) super.run - Serializer.serialize(ctx.base.coverage.nn, outputPath, ctx.settings.sourceroot.value) + val mergedCoverage = Coverage() + + previousCoverage.statements + .filterNot(stmt => + val source = stmt.location.sourcePath + lastCompiledFiles.contains(source.toString) || !Files.exists(source) + ) + .foreach { stmt => + mergedCoverage.addStatement(stmt) + } + ctx.base.coverage.nn.statements.foreach(stmt => mergedCoverage.addStatement(stmt)) + + Serializer.serialize(mergedCoverage, outputPath, ctx.settings.sourceroot.value) private def isClassIncluded(sym: Symbol)(using Context): Boolean = val fqn = sym.fullName.toText(ctx.printerFn(ctx)).show @@ -253,6 +273,7 @@ class InstrumentCoverage extends MacroTransform with IdentityDenotTransformer: InstrumentedParts.singleExprTree(coverageCall, transformed) override def transform(tree: Tree)(using Context): Tree = + lastCompiledFiles += tree.sourcePos.source.file.absolute.jpath.toString inContext(transformCtx(tree)) { // necessary to position inlined code properly tree match // simple cases From 90b264b3e9c7ae7664ca22ee818d2993e4a0afec Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Thu, 18 Dec 2025 17:05:19 +0100 Subject: [PATCH 2/3] Add tests and fix implementation --- .../dotty/tools/dotc/coverage/Coverage.scala | 12 ++-- .../tools/dotc/coverage/Serializer.scala | 10 +-- .../dotc/transform/InstrumentCoverage.scala | 4 +- .../tools/dotc/coverage/CoverageTests.scala | 66 +++++++++++++++++++ 4 files changed, 80 insertions(+), 12 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/coverage/Coverage.scala b/compiler/src/dotty/tools/dotc/coverage/Coverage.scala index 7df2e503e3f4..881a9e8a58d3 100644 --- a/compiler/src/dotty/tools/dotc/coverage/Coverage.scala +++ b/compiler/src/dotty/tools/dotc/coverage/Coverage.scala @@ -8,16 +8,16 @@ import java.nio.file.Path class Coverage: private val statementsById = new mutable.LongMap[Statement](256) - private var statementId: Int = 0 - - def nextStatementId(): Int = - statementId += 1 - statementId - 1 + private var _nextStatementId: Int = 1 + def nextStatementId(): Int = _nextStatementId + def setNextStatementId(id: Int): Unit = _nextStatementId = id def statements: Iterable[Statement] = statementsById.values - def addStatement(stmt: Statement): Unit = statementsById(stmt.id) = stmt + def addStatement(stmt: Statement): Unit = + if stmt.id >= _nextStatementId then _nextStatementId = stmt.id + 1 + statementsById(stmt.id) = stmt def removeStatementsFromFile(sourcePath: Path | Null) = val removedIds = statements.filter(_.location.sourcePath == sourcePath).map(_.id.toLong) diff --git a/compiler/src/dotty/tools/dotc/coverage/Serializer.scala b/compiler/src/dotty/tools/dotc/coverage/Serializer.scala index d46ef46479df..91406570081b 100644 --- a/compiler/src/dotty/tools/dotc/coverage/Serializer.scala +++ b/compiler/src/dotty/tools/dotc/coverage/Serializer.scala @@ -16,7 +16,7 @@ object Serializer: private val CoverageFileName = "scoverage.coverage" private val CoverageDataFormatVersion = "3.0" - def coverageFilePath(dataDir: String) = + def coverageFilePath(dataDir: String): Path = Paths.get(dataDir, CoverageFileName).toAbsolutePath /** Write out coverage data to the given data directory, using the default coverage filename */ @@ -90,12 +90,12 @@ object Serializer: .sortBy(_.id) .foreach(stmt => writeStatement(stmt, writer)) - def deserialize(file: Path): Coverage = + def deserialize(file: Path, sourceRoot: String): Coverage = val source = Source.fromFile(file.toFile(), UTF_8.name()) - try deserialize(source.getLines()) + try deserialize(source.getLines(), Paths.get(sourceRoot).toAbsolutePath) finally source.close() - def deserialize(lines: Iterator[String]): Coverage = + def deserialize(lines: Iterator[String], sourceRoot: Path): Coverage = def toStatement(lines: Iterator[String]): Statement = val id: Int = lines.next().toInt val sourcePath = lines.next() @@ -110,7 +110,7 @@ object Serializer: fullClassName, classType, method, - Paths.get(sourcePath) + sourceRoot.resolve(sourcePath) ) val start: Int = lines.next().toInt val end: Int = lines.next().toInt diff --git a/compiler/src/dotty/tools/dotc/transform/InstrumentCoverage.scala b/compiler/src/dotty/tools/dotc/transform/InstrumentCoverage.scala index a10c2a276a74..9510e724f178 100644 --- a/compiler/src/dotty/tools/dotc/transform/InstrumentCoverage.scala +++ b/compiler/src/dotty/tools/dotc/transform/InstrumentCoverage.scala @@ -61,7 +61,7 @@ class InstrumentCoverage extends MacroTransform with IdentityDenotTransformer: val coverageFilePath = Serializer.coverageFilePath(outputPath) val previousCoverage = if Files.exists(coverageFilePath) then - Serializer.deserialize(coverageFilePath) + Serializer.deserialize(coverageFilePath, ctx.settings.sourceroot.value) else Coverage() // Initialise a coverage object if it does not exist yet @@ -72,6 +72,8 @@ class InstrumentCoverage extends MacroTransform with IdentityDenotTransformer: coverageExcludeFilePatterns = ctx.settings.coverageExcludeFiles.value.map(_.r.pattern) ctx.base.coverage.nn.removeStatementsFromFile(ctx.compilationUnit.source.file.absolute.jpath) + ctx.base.coverage.nn.setNextStatementId(previousCoverage.nextStatementId()) + super.run val mergedCoverage = Coverage() diff --git a/compiler/test/dotty/tools/dotc/coverage/CoverageTests.scala b/compiler/test/dotty/tools/dotc/coverage/CoverageTests.scala index cd16d3d536a1..936831cff140 100644 --- a/compiler/test/dotty/tools/dotc/coverage/CoverageTests.scala +++ b/compiler/test/dotty/tools/dotc/coverage/CoverageTests.scala @@ -18,6 +18,7 @@ import scala.language.unsafeNulls import scala.collection.mutable.Buffer import dotty.tools.dotc.util.DiffUtil +import java.nio.charset.StandardCharsets import java.util.stream.Collectors @Category(Array(classOf[BootstrappedOnlyTests])) @@ -127,6 +128,71 @@ class CoverageTests: ) } + @Test + def checkIncrementalCoverage(): Unit = + val target = Files.createTempDirectory("coverage") + val sourceRoot = target.resolve("src") + Files.createDirectory(sourceRoot) + val sourceFile1 = sourceRoot.resolve("file1.scala") + Files.write(sourceFile1, "def file1() = 1".getBytes(StandardCharsets.UTF_8)) + + val coverageOut = target.resolve("coverage-out") + Files.createDirectory(coverageOut) + val options = defaultOptions.and("-Ycheck:instrumentCoverage", "-coverage-out", coverageOut.toString, "-sourceroot", sourceRoot.toString) + compileFile(sourceFile1.toString, options).checkCompile() + + val scoverageFile = coverageOut.resolve("scoverage.coverage") + assert(Files.exists(scoverageFile), s"Expected scoverage file to exist at $scoverageFile") + + locally { + val coverage = Serializer.deserialize(scoverageFile, sourceRoot.toString()) + val filesWithCoverage = coverage.statements.map(_.location.sourcePath.getFileName.toString).toSet + assertEquals(Set("file1.scala"), filesWithCoverage) + } + + val sourceFile2 = sourceRoot.resolve("file2.scala") + Files.write(sourceFile2, "def file2() = 2".getBytes(StandardCharsets.UTF_8)) + + compileFile(sourceFile2.toString, options).checkCompile() + locally { + val coverage = Serializer.deserialize(scoverageFile, sourceRoot.toString()) + val filesWithCoverage = coverage.statements.map(_.location.sourcePath.getFileName.toString).toSet + assertEquals(Set("file1.scala", "file2.scala"), filesWithCoverage) + } + + @Test + def `deleted source files should not be kept in incremental coverage`(): Unit = + val target = Files.createTempDirectory("coverage") + val sourceRoot = target.resolve("src") + Files.createDirectory(sourceRoot) + val sourceFile1 = sourceRoot.resolve("file1.scala") + Files.write(sourceFile1, "def file1() = 1".getBytes(StandardCharsets.UTF_8)) + + val coverageOut = target.resolve("coverage-out") + Files.createDirectory(coverageOut) + val options = defaultOptions.and("-Ycheck:instrumentCoverage", "-coverage-out", coverageOut.toString, "-sourceroot", sourceRoot.toString) + compileFile(sourceFile1.toString, options).checkCompile() + + val scoverageFile = coverageOut.resolve("scoverage.coverage") + assert(Files.exists(scoverageFile), s"Expected scoverage file to exist at $scoverageFile") + + locally { + val coverage = Serializer.deserialize(scoverageFile, sourceRoot.toString()) + val filesWithCoverage = coverage.statements.map(_.location.sourcePath.getFileName.toString).toSet + assertEquals(Set("file1.scala"), filesWithCoverage) + } + + val sourceFile2 = sourceRoot.resolve("file2.scala") + Files.write(sourceFile2, "def file2() = 2".getBytes(StandardCharsets.UTF_8)) + + Files.delete(sourceFile1) + + compileFile(sourceFile2.toString, options).checkCompile() + locally { + val coverage = Serializer.deserialize(scoverageFile, sourceRoot.toString()) + val filesWithCoverage = coverage.statements.map(_.location.sourcePath.getFileName.toString).toSet + assertEquals(Set("file2.scala"), filesWithCoverage) + } object CoverageTests extends ParallelTesting: import scala.concurrent.duration.* From 474948a50e783b0c941362c341fa26479e4aa7e8 Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Fri, 19 Dec 2025 13:18:54 +0100 Subject: [PATCH 3/3] Fixes --- compiler/src/dotty/tools/dotc/coverage/Serializer.scala | 2 +- .../src/dotty/tools/dotc/transform/InstrumentCoverage.scala | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/coverage/Serializer.scala b/compiler/src/dotty/tools/dotc/coverage/Serializer.scala index 91406570081b..21d3a83cb3a0 100644 --- a/compiler/src/dotty/tools/dotc/coverage/Serializer.scala +++ b/compiler/src/dotty/tools/dotc/coverage/Serializer.scala @@ -110,7 +110,7 @@ object Serializer: fullClassName, classType, method, - sourceRoot.resolve(sourcePath) + sourceRoot.resolve(sourcePath).normalize() ) val start: Int = lines.next().toInt val end: Int = lines.next().toInt diff --git a/compiler/src/dotty/tools/dotc/transform/InstrumentCoverage.scala b/compiler/src/dotty/tools/dotc/transform/InstrumentCoverage.scala index 9510e724f178..e0878f34ca22 100644 --- a/compiler/src/dotty/tools/dotc/transform/InstrumentCoverage.scala +++ b/compiler/src/dotty/tools/dotc/transform/InstrumentCoverage.scala @@ -275,7 +275,9 @@ class InstrumentCoverage extends MacroTransform with IdentityDenotTransformer: InstrumentedParts.singleExprTree(coverageCall, transformed) override def transform(tree: Tree)(using Context): Tree = - lastCompiledFiles += tree.sourcePos.source.file.absolute.jpath.toString + val path = tree.sourcePos.source.file.absolute.jpath + if path != null then lastCompiledFiles += path.toString + inContext(transformCtx(tree)) { // necessary to position inlined code properly tree match // simple cases